diff --git a/.agents/skills/fix-and-pr/SKILL.md b/.agents/skills/fix-and-pr/SKILL.md new file mode 100644 index 00000000..5122e5d2 --- /dev/null +++ b/.agents/skills/fix-and-pr/SKILL.md @@ -0,0 +1,190 @@ +--- +name: fix-and-pr +description: Reproduce, debug, fix, verify, and submit GBOS/Eonix kernel issues as pull requests. Use for kernel bug fixing, headless QEMU reproduction scripts, subagent-driven root-cause analysis, commit preparation, and credential-safe PR creation. +--- + +# GBOS / Eonix Fix and PR + +## Mandatory credential gate + +Run this before **any** other action: preflight, branch creation, issue +fetching, builds, analysis, or subagent launches. + +1. Check `GH_TOKEN` without printing it: + + ```bash + test -n "${GH_TOKEN:-}" + ``` + +2. If the check succeeds, continue. +3. If `GH_TOKEN` is unset or empty, stop immediately. Ask the user for a GitHub + token, preferably by exporting it: + + ```text + GH_TOKEN is not set. Please provide a GitHub token by exporting it in your + shell, for example: + export GH_TOKEN='...' + Tell me when it is set; I will not continue until then. + ``` + +4. Do not run tools, inspect the repository, spawn subagents, or begin the + workflow until the user confirms it is available and the check succeeds. +5. Never echo, log, commit, or put `GH_TOKEN` in a subagent prompt. Do not + write it to a file. + +The detailed protocol is split by phase under `references/`. Read +`references/README.md`, then `references/00-operating-contract.md`, then the +document for the current phase. Load only the documents needed for that phase. + +If the task is not a GBOS/Eonix kernel issue, keep the same phase gates but +replace repository-specific paths, build commands, and remotes. + +## When this skill applies + +- A GitHub issue, bug report, stack trace, or failing test must be reproduced + and fixed. +- A QEMU-based kernel reproduction script is needed. +- The work needs subagent orchestration for analysis, implementation, review, + or PR publication. +- The user asks to prepare commits or open a PR for a verified fix. + +## Entry point + +```bash +ISSUE= +SLUG= +BASE= +ART="build/work/issue-${ISSUE}" +mkdir -p "$ART" +``` + +Then: + +1. Confirm the credential gate above passes. +2. Read `references/README.md` and `references/00-operating-contract.md`. +3. Run the repository preflight in `references/01-environment-and-intake.md`. +4. Capture the issue in `$ART/issue.json`. +5. Follow the phase gates below in order, loading the matching reference + document when entering each phase. + +## Phase gates + +| Phase | Required evidence | Reference document | +| --- | --- | --- | +| 0. Intake and environment | `GH_TOKEN` present; clean checkout, configured toolchain, successful build, target QEMU boot, issue details captured. | [01 — Environment and issue intake](references/01-environment-and-intake.md) | +| 1. Reproduction | Headless script independently verified to distinguish bug from fixed behavior. | [02 — Automated reproduction](references/02-reproduction.md) | +| 2. Analysis | Root cause stated with evidence; fix boundary and invariants recorded. | [03 — Root-cause analysis](references/03-analysis.md) | +| 3. Fix and commits | Minimal scoped fix; clean, independently buildable commits with repository-style messages. | [04 — Fix planning and implementation](references/04-fix-and-commits.md), [05 — Commit conventions](references/05-commit-conventions.md) | +| 4. Verification and review | Baseline `--expect bug` passes, fixed `--expect fixed` passes, builds/tests pass, independent review has no blockers. | [06 — Verification and review](references/06-verification.md) | +| 5. PR | Correct base/head, pushed branch, PR body with verification evidence, local and PR diffs match, no secrets. | [07 — PR submission](references/07-pr-submission.md) | + +Do not advance because a plausible fix exists. Advance only when the gate +evidence exists. + +## Critical rules + +1. **One writer per worktree.** Never run concurrent writers in the same + checkout. +2. **No reproduction, no fix.** Do not implement a speculative fix. +3. **No post-fix verification, no PR.** The same reproducer must pass in fixed + mode after the fix. +4. **Never use `git add -A` or `git add .`.** Add explicit paths only. +5. **Never leak `GH_TOKEN`.** Do not print it, pass it in prompts, write it to + disk, embed it in URLs, or commit it. +6. **Keep the fix minimal.** No unrelated refactors, formatting sweeps, or + dependency updates. +7. **Separate authoring from review.** Use a fresh-context `reviewer` for the + independent diff review. +8. **Stop on ambiguity.** Ask before changing unspecified behavior or choosing + a PR base branch. + +## Reproduction contract + +Save the script at: + +```text +script/repro/issue--.exp +``` + +It must support: + +```text +--expect bug|fixed +--arch +--mode +--smp +--timeout +--log +--help +``` + +Exit codes: + +| Code | Meaning | +| --- | --- | +| `0` | Observed behavior matched `--expect` | +| `1` | Observed behavior contradicted `--expect` | +| `2` | Infrastructure failure | +| `3` | Usage error | +| `130` | Interrupted | + +Use `make test-run` for headless execution. Give QEMU a unique name and kill +only that instance; never use global `killall qemu-system-*`. + +Before trusting the script, have a fresh verifier confirm: + +- `--expect bug` exits 0 on the unmodified tree. +- `--expect fixed` exits 1 on the unmodified tree (negative control). +- No QEMU process remains after any exit path. +- Logs contain no secrets and are bounded. + +## Supporting references + +- [08 — Failure handling](references/08-failure-handling.md) +- [09 — Issue analysis template](references/09-analysis-template.md) +- [10 — Quick command reference](references/10-command-reference.md) +- [11 — Prompt hygiene](references/11-prompt-hygiene.md) + +## Artifacts + +Keep issue state outside chat history: + +```text +build/work/issue-/ +├── issue.json +├── analysis.md +├── plan.md +├── script-review.md +├── before.log +├── after.log +└── pr.md +``` + +`build/` is ignored by Git, but `make clean` deletes it. Preserve important logs +elsewhere before a clean rebuild. + +## Subagent roles + +| Role | Use | +| --- | --- | +| `scout` | Read-only issue intake, code mapping, log analysis, hypothesis generation. | +| `worker` | One bounded implementation or script-generation task in one worktree. | +| `reviewer` | Fresh-context challenge of the reproduction script, root cause, plan, or final diff. | +| Publisher `worker` | Push the verified branch and create the PR; no code changes. | + +The supervisor retains phase-gate decisions, final diff ownership, and +publication authority. + +## Output expectations + +When reporting progress or completion, include: + +- Issue id and chosen base/head branches. +- Root cause and changed files. +- Baseline and fixed reproduction results with log paths. +- Build/test commands and exit codes. +- Independent review verdict. +- Commit hashes and PR URL, if created. +- Remaining risks or blockers. + +READ ALL REFERENCE DOCUMENTATIONS UNDER THIS SKILL'S references/ DIRECTORY BEFORE CONTINUING! diff --git a/.agents/skills/fix-and-pr/references/00-operating-contract.md b/.agents/skills/fix-and-pr/references/00-operating-contract.md new file mode 100644 index 00000000..636e9f48 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/00-operating-contract.md @@ -0,0 +1,121 @@ +# 00 — Operating contract + +Part of the GBOS/Eonix `fix-and-pr` skill. + +A standardized, end-to-end workflow for reproducing kernel issues, finding the +root cause, implementing and verifying a fix, and opening a pull request with +subagents. + +The workflow is deliberately **evidence-gated**. Do not advance to the next +phase because a plausible fix exists; advance only when the previous phase has +produced verifiable evidence. + +--- + +## 0. Operating contract + +### 0.1 Roles + +| Role | Owner | Responsibility | +| --- | --- | --- | +| Supervisor | Parent agent | Owns issue interpretation, phase gates, final diff, commits, and PR. Resolves ambiguity and approves publication. | +| Scout | `scout` subagent (read-only) | Fetches issue context, maps code paths, collects logs, proposes hypotheses. | +| Worker | `worker` subagent (writer) | Implements one bounded change in one worktree. | +| Reviewer | `reviewer` subagent (read-only, fresh context) | Independently challenges the reproduction script, diff, or root cause. | +| Verifier | `worker` or `reviewer` with a test-only mandate | Runs the reproduction and builds; does not edit kernel code. | +| Publisher | `worker` subagent | Pushes the branch and creates the PR. Does not change code. | + +The supervisor may perform any step directly when that is cheaper than +delegation. Subagents are used to obtain isolation, independent verification, +and parallelism, not as ceremony. + +### 0.2 Mandatory credential gate + +Before any other workflow action, check `GH_TOKEN` without printing it: + +```bash +test -n "${GH_TOKEN:-}" +``` + +- If it is set and non-empty, continue. +- If it is unset or empty, stop and ask the user for the token. Prefer asking + them to export it in their shell: + + ```text + GH_TOKEN is not set. Please provide a GitHub token by exporting it: + export GH_TOKEN='...' + Tell me when it is set; I will not continue until then. + ``` + +- Do not inspect the repository, fetch issues, build, spawn subagents, or begin + any phase until the user confirms it is available and the check succeeds. +- Never echo, log, commit, or put `GH_TOKEN` in a prompt. Do not write it to a + file. + +### 0.3 Non-negotiable rules + +1. **One writer per worktree.** Never let two subagents edit the same checkout + concurrently. +2. **No fix without reproduction.** If the issue cannot be reproduced, record + why and stop; do not implement a speculative fix. +3. **No PR without post-fix verification.** The same reproduction that failed + before the fix must pass after it. +4. **Separate evidence from authority.** A subagent report is evidence. The + supervisor decides whether it is sufficient. +5. **Never leak credentials.** `GH_TOKEN` is read from the environment only. + Never print it, put it in a URL, a prompt, a commit, a log, or a file. +6. **Never use `git add -A` or `git add .`.** Explicitly add intended files; + otherwise an untracked secret or log can enter the commit. +7. **Minimal scope.** No unrelated refactors, formatting sweeps, or speculative + improvements in a bug-fix PR. +8. **Stop on ambiguity.** Ask before changing behavior that is not clearly + required by the issue, or when the target base branch is unclear. + +### 0.4 Phase gates + +| Phase | Gate to leave the phase | +| --- | --- | +| 0. Intake and environment | `GH_TOKEN` is present; repository builds, target architecture runs, issue details captured. | +| 1. Reproduction | A headless script exists and has been independently verified to distinguish bug from fixed behavior. | +| 2. Analysis | Root cause is stated with evidence; the fix boundary is identified. | +| 3. Fix and commits | Minimal fix is implemented; commits are clean and build independently. | +| 4. Verification and review | Reproduction passes in fixed mode, baseline passes in bug mode, builds/tests pass, independent review has no blocking findings. | +| 5. PR | Branch is pushed, PR is created with the correct base/head and verification evidence, and the local diff matches the PR diff. | + +### 0.5 Persistent artifacts + +Use issue-specific paths so later subagents can resume without relying on chat +history: + +```bash +ISSUE=123 +SLUG=short-issue-slug +ART="build/work/issue-${ISSUE}" +mkdir -p "$ART" +``` + +```text +build/work/issue-/ +├── issue.json # issue metadata and comments, if fetched +├── analysis.md # observations, hypotheses, root cause +├── plan.md # fix plan and test matrix +├── script-review.md # independent reproduction-script verdict +├── before.log # --expect bug run on the unmodified tree +├── after.log # --expect fixed run on the fixed tree +└── pr.md # final PR title/body and URL +``` + +Reproduction scripts belong in `script/repro/`, because they may be reused by +later subagents and optionally committed as regression tests: + +```text +script/repro/issue--.exp +``` + +`build/` is ignored by Git, but `make clean` deletes it. Copy important logs +out before a clean rebuild, or keep the canonical logs in a temporary directory +such as `/tmp/gbos-issue-/`. + +--- + +Next: [01 — Environment and issue intake](01-environment-and-intake.md) diff --git a/.agents/skills/fix-and-pr/references/01-environment-and-intake.md b/.agents/skills/fix-and-pr/references/01-environment-and-intake.md new file mode 100644 index 00000000..d9bbac5a --- /dev/null +++ b/.agents/skills/fix-and-pr/references/01-environment-and-intake.md @@ -0,0 +1,152 @@ +# 01 — Environment and issue intake + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 1. Phase 0: Intake, environment, and target architecture + +### 1.1 Credential gate and repository preflight + +Before running any repository command, confirm the mandatory credential gate +from [00 — Operating contract](00-operating-contract.md) has passed: + +```bash +test -n "${GH_TOKEN:-}" +``` + +If it fails, stop and ask the user to provide/export `GH_TOKEN`. Do not run the +commands below, spawn subagents, or start the workflow until it succeeds. + +After the gate passes, run this repository preflight: + +```bash +pwd +git status --short --branch +git remote -v +git branch --show-current +git log --oneline -10 +``` + +Requirements: + +- The checkout is on the intended base branch or a dedicated fix branch. +- There are no unintended local changes. Untracked workflow artifacts are fine + only when they are known and will not be committed. +- `GH_TOKEN` is present in the environment and has not been written to the + repository or any workflow artifact. +- The remote used for the PR is known: + - `github` often points at the upstream mirror or fork. + - `origin` may point at a private/self-hosted remote. + - Do not assume which remote is the PR target; confirm it in Phase 5. + +### 1.2 Build and runtime environment + +The kernel primarily targets **x86_64** (default target +`x86_64-acpi_compat`), with cross-support for `riscv64` and `loongarch64`. + +Key build parameters: + +| Variable | Meaning | Default | +| --- | --- | --- | +| `ARCH` | Target architecture: `x86_64`, `riscv64`, `loongarch64` | `x86_64` | +| `MODE` | Build profile: `release` or `debug` | `debug` | +| `SMP` | Number of virtual CPUs; empty disables the `smp` feature | `4` | +| `FEATURES` | Comma-separated Cargo features, e.g. `log_debug,trace_syscall` | empty | +| `QEMU_ACCEL` | QEMU accelerator; `-accel tcg` on macOS, `-accel kvm` on Linux when available | auto | +| `DEBUG_TRAPS` | Enables QEMU trap logging to `build/qemu.log` | unset | + +Configure and build: + +```bash +./configure +make build MODE=release +``` + +If `./configure` cannot find a required tool, fix the environment before +continuing. `./configure` generates the `Makefile`; the generated file is +ignored by Git. + +Interactive run: + +```bash +make run MODE=release +``` + +Wait for the shell prompt (`/ # `), run a harmless command such as +`echo READY`, then exit QEMU with `Ctrl-A`, then `x`. + +Headless/test run: + +```bash +make test-run MODE=release ARCH=x86_64 +``` + +`make test-run` uses `-serial stdio` and is the preferred target for automated +reproduction. Unlike `make run`, it does not multiplex the QEMU monitor into +the serial stream, so exit by terminating the QEMU process rather than +`Ctrl-A x`. + +Debug run: + +```bash +make srun MODE=release # starts QEMU paused with GDB stub on :1234 +make debug # attaches GDB and loads build/kernel.sym +# or, when tmux is available: +make tmux-debug +``` + +### 1.3 Create a fix branch + +Use an isolated worktree when other work may continue in the main checkout: + +```bash +git fetch github +git switch -c "fix/${ISSUE}-${SLUG}" "github/master" +# or: +git worktree add -b "fix/${ISSUE}-${SLUG}" "../eonix-${ISSUE}" "github/master" +``` + +If the issue targets a feature branch, branch from that branch instead of +`master`. Record the chosen base in `build/work/issue-/plan.md`. + +### 1.4 Capture the issue + +Preferred, if `gh` is installed: + +```bash +gh issue view "$ISSUE" \ + --repo greatbridf/osdev \ + --json number,title,body,labels,comments \ + > "$ART/issue.json" +``` + +`GH_TOKEN` is the standard environment variable read by the `gh` CLI and by the +REST fallback. Export it once for the workflow; do not write it to a file. + +Fallback with the GitHub REST API: + +```bash +curl -sS \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/greatbridf/osdev/issues/${ISSUE}" \ + > "$ART/issue.json" +``` + +Do not put the token in shell history or command output. If a command would +print it, redirect the output or redact the value. + +Capture in `analysis.md`: + +- Expected behavior vs. actual behavior. +- Reproducer commands, stack traces, logs, and screenshots referenced by the + issue. +- Target architecture, `MODE`, `SMP`, and whether the bug is SMP-only. +- Whether the issue is a hang, panic, wrong result, memory corruption, race, or + build failure. +- The commit or branch the reporter used, if known. + +--- + +Next: [02 — Automated reproduction](02-reproduction.md) diff --git a/.agents/skills/fix-and-pr/references/02-reproduction.md b/.agents/skills/fix-and-pr/references/02-reproduction.md new file mode 100644 index 00000000..ec2119f8 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/02-reproduction.md @@ -0,0 +1,219 @@ +# 02 — Automated reproduction + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 2. Phase 1: Automated reproduction + +Manual interactive QEMU sessions easily hang terminals or leave orphaned +processes. Reproduce with a script, and verify the script itself before trusting +its result. + +### 2.1 Spawn a subagent to generate the reproduction script + +Spawn a `scout` or `worker` with a bounded, script-only mandate. The subagent +may inspect the repository, but must not edit kernel source. + +#### Generator task template + +```text +Objective +Create an automated, headless reproduction for GBOS/Eonix issue #: +. + +Repository and target +- cwd: +- ref/branch: +- target: ARCH=, MODE=release, SMP= +- issue evidence: + +Authority and boundaries +- You may create or edit only script/repro/issue--.exp. +- Do not edit kernel code, user programs, the Makefile, or repository docs. +- Do not commit. +- Do not print, copy, or commit any credential. + +Required script interface +The script must accept: + --expect bug|fixed required; a default of "bug" is acceptable if documented + --arch default x86_64 + --mode default release + --smp default 4 + --timeout overall budget, default 180 + --log default build/work/issue-/.log + --help print usage and exit without launching QEMU + +Required behavior +1. Validate arguments before launching anything. +2. Launch the kernel with `make test-run` (not `make run`), using + `ARCH`, `MODE`, and `SMP` from the arguments. +3. Give QEMU a unique name, e.g. `-name gbos-repro-`, via QEMU_ARGS, so + cleanup can target this instance only. +4. Wait for the busybox shell prompt `/ # ` and fail as infrastructure error + if it never appears. +5. Send the exact command(s) that trigger the issue. +6. Detect success, wrong output, panic, hang, or QEMU exit with explicit + timeouts. Treat a kernel panic or a timeout after the trigger as "bug + observed", not as infrastructure failure. +7. On completion, terminate only the QEMU instance started by this script — + never `killall qemu-system-*`. Kill the process group or use the unique + QEMU name. Ensure no orphan remains. +8. Write a bounded log; do not let it grow without limit. +9. Exit with the contract below. + +Exit-code contract +0 observed behavior matches --expect +1 observed behavior contradicts --expect +2 infrastructure failure: build failure, no prompt, missing tool, cleanup + failure, or timeout before the trigger could be sent +3 usage/argument error +130 interrupted by the user + +Required log markers +REPRO issue= expect= +REPRO arch= mode= smp= +REPRO booted +REPRO trigger-sent +REPRO bug-observed | REPRO fixed-observed | REPRO timeout +REPRO result=pass|fail exit= log= + +Acceptance +- `--help` exits 0 without starting QEMU. +- On the unmodified tree, `--expect bug` exits 0 and the log shows the issue. +- On the unmodified tree, `--expect fixed` exits non-zero with a mismatch, not + a false success. +- No QEMU process started by the script remains after any exit path. +- The script contains no credentials and no absolute paths tied to your home + directory. + +Deliverable +- The script path. +- The exact command used for a baseline run. +- The baseline log path and exit code. +- Any assumptions or flakiness observed. +``` + +Use `expect` or Python `pexpect`. Existing `script/test.sh` is a useful +reference for prompt matching, timeouts, and `poweroff`, but its global +`killall` cleanup is not acceptable for a reusable reproducer. + +### 2.2 Spawn a subagent to verify the reproduction script + +Do **not** immediately trust the generator. A broken script can "pass" for the +wrong reason or fail on infrastructure. The verifier must use a fresh context +and must not modify kernel code. + +#### Verifier task template + +```text +Objective +Independently verify that script/repro/issue--.exp is a trustworthy +reproduction for issue #. Do not fix the kernel. + +Repository and ref +- cwd: +- ref: +- artifacts: build/work/issue-/ + +Authority and boundaries +- Read-only for kernel code. +- You may edit only the reproduction script if it is defective, and you must + report every change. Do not edit tests, Makefile, or kernel sources. +- Do not commit, push, or create a PR. +- Never print credentials. + +Verification steps +1. Read the issue and the script. Confirm the script tests the reported behavior + rather than an unrelated proxy. +2. Run `script/repro/issue--.exp --help`; confirm it exits 0 and starts + no QEMU. +3. On the unmodified tree, run `--expect bug` in the documented configuration. + Confirm exit 0 and that the log contains the trigger and a bug/timeout/panic + marker. +4. Run the negative control on the same unmodified tree: + `--expect fixed`. It must exit 1 (mismatch) and must not report a false + success. If it exits 0, the script cannot distinguish bug from fixed. +5. Check the timeout behavior: the script must fail as infrastructure (2) if the + prompt never appears, and report "bug observed" (0 for --expect bug) if the + trigger hangs. +6. Check cleanup after every run: + `pgrep -fl 'qemu-system-.*gbos-repro-'` must return nothing. +7. Check that logs do not contain credentials, home-directory paths, or unbounded + output. +8. If the bug is timing-dependent, repeat the baseline at least 3 times and + report the pass rate. Do not hide flakiness. + +Report +Verdict: PASS | FAIL +- Script path and exact commands run. +- Baseline --expect bug: exit code, log path. +- Negative control --expect fixed: exit code, log path. +- Cleanup check: command and result. +- Flakiness: number of runs and pass rate. +- Defects found; if you edited the script, show the diff and explain why. +- Remaining uncertainty. +``` + +If the verifier cannot make the script distinguish bug from fixed behavior, +return to the generator. Do not proceed to analysis with an ambiguous test. + +### 2.3 Running and reusing the script + +Use the same script and configuration throughout the workflow. + +```bash +# On the unmodified tree: confirm the bug is present. +script/repro/issue-${ISSUE}-${SLUG}.exp \ + --expect bug \ + --arch x86_64 --mode release --smp 4 \ + --log "$ART/before.log" + +# After the fix and rebuild: confirm the bug is gone. +script/repro/issue-${ISSUE}-${SLUG}.exp \ + --expect fixed \ + --arch x86_64 --mode release --smp 4 \ + --log "$ART/after.log" +``` + +Interpretation: + +| Exit | Meaning | +| --- | --- | +| `0` | Observed behavior matched `--expect`; this is a successful run. | +| `1` | Behavior contradicted `--expect`; the fix is incomplete or the baseline did not reproduce. | +| `2` | Infrastructure failure; fix the environment before drawing conclusions. | +| `3` | Invalid invocation; fix the command. | + +The exact same script and arguments must be used before and after the fix. If +the script must change, re-run both baseline and fixed cases with the new +version and record why it changed. + +For SMP or race-related issues, run a matrix and keep all logs: + +```bash +for smp in 1 4; do + for i in 1 2 3; do + script/repro/issue-${ISSUE}-${SLUG}.exp \ + --expect bug --smp "$smp" \ + --log "$ART/before-smp${smp}-run${i}.log" + done +done +``` + +### 2.4 If the issue cannot be reproduced + +Stop and report: + +- The exact commands and environment tried. +- The arch, `MODE`, `SMP`, and QEMU accelerator used. +- Whether the issue depends on a feature branch, a user program, an additional + disk image (`IMG=`), or a particular init script. +- The closest observed behavior and its log. + +Ask the issue reporter or supervisor for missing details. Do not invent a +reproduction by weakening the assertion or by testing a different behavior. + +--- + +Next: [03 — Root-cause analysis and debugging](03-analysis.md) diff --git a/.agents/skills/fix-and-pr/references/03-analysis.md b/.agents/skills/fix-and-pr/references/03-analysis.md new file mode 100644 index 00000000..e279bfa5 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/03-analysis.md @@ -0,0 +1,148 @@ +# 03 — Root-cause analysis and debugging + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 3. Phase 2: Root-cause analysis and debugging + +The goal is a **causal explanation**, not a patch that happens to make the test +green. + +### 3.1 Freeze the evidence + +Before editing anything: + +```bash +git rev-parse HEAD > "$ART/baseline-commit.txt" +git status --short > "$ART/baseline-status.txt" +cp "$ART/before.log" "$ART/before.keep.log" +``` + +Preserve: + +- The baseline commit and branch. +- The exact reproduction command and configuration. +- The complete panic/backtrace text. +- The QEMU log, if trap logging was enabled. +- Any kernel symbols or disassembly used. + +For trap logging: + +```bash +DEBUG_TRAPS=1 make test-run MODE=release ARCH=x86_64 +# QEMU writes detailed logs to build/qemu.log +``` + +### 3.2 Static reconnaissance + +Map the code path before changing it: + +```bash +rg -n "" src crates macros +rg -n "" src crates +git log --oneline --all -- +git blame -L , -- +``` + +Inspect: + +- Callers and callees of the failing function. +- Lock ordering, reference counting, and lifetime assumptions. +- Error paths: allocation failure, invalid user pointers, partial initialization. +- Architecture-specific implementations under `src/` and `crates/eonix_hal`. +- Related filesystem, scheduler, VFS, MM, or syscall code. +- Existing tests and documentation under `doc/` and `script/`. + +For GBOS/Eonix, explicitly consider: + +- **SMP:** Is the code correct with concurrent access? Does it assume a single + CPU? Is the lock held across an await point or a blocking operation? +- **Interrupts/preemption:** Can the state change between check and use? +- **Refcounts:** Who owns each reference, and which path drops it? +- **Page tables/folios:** Are mappings, CoW, and page flags updated atomically? +- **User memory:** Is the access validated, and can it fault mid-copy? +- **Architecture:** Does the fix need a RISC-V or LoongArch counterpart? +- **ABI:** Does the change alter syscall return values, errno, or struct layout? + +### 3.3 Dynamic analysis + +Use the smallest tool that answers the next question. + +- **QEMU trap log:** `DEBUG_TRAPS=1` and inspect `build/qemu.log`. +- **GDB:** `make srun` then `make debug`, or `make tmux-debug`. +- **Backtrace:** `bt full`, `thread apply all bt`, `info registers`. +- **Memory/state:** `x/`, `p`, `p/x`, `info threads`, `monitor info registers`. +- **Tracing:** enable `trace_syscall`, `trace_scheduler`, `trace_pci`, or + `log_debug` only as needed. Remove or feature-gate temporary prints before + committing. +- **SMP comparison:** run `SMP=1` and `SMP=4`; a difference is strong evidence + of a race or missing synchronization. +- **Architecture comparison:** if x86_64 fails, check whether the shared code + or the HAL is responsible; test a second architecture when feasible. + +Do not leave GDB or QEMU running between phases. Record the exact commands and +observations in `analysis.md`. + +### 3.4 Form and disprove hypotheses + +Write down candidates before testing: + +```text +Hypothesis: +Expected behavior: +Actual behavior: +Evidence for: +Evidence against: +Falsification test: +``` + +Rules: + +- Prefer the hypothesis that explains the **first** wrong state, not the last + visible symptom. +- Distinguish the bug from the mechanism that exposes it. +- If a hypothesis predicts a log line, register value, or code path, verify it + directly. +- If two independent hypotheses remain, run one read-only `scout` per + hypothesis; do not run duplicate scouts with the same prompt. +- Keep failed hypotheses in `analysis.md`; they prevent the next agent from + repeating the work. + +### 3.5 Write the root-cause statement + +Before planning the fix, complete this sentence with evidence: + +```text +Root cause: causes because , as shown by +. +``` + +Also record: + +- The smallest code boundary that must change. +- What must **not** change. +- Which invariants the fix must preserve. +- Which tests or configurations could regress. + +If the root cause is not established, do not proceed to Phase 3. + +### 3.6 Optional independent analysis + +For a high-risk or surprising diagnosis, spawn a fresh `reviewer` with the +evidence and ask it to challenge the conclusion: + +```text +Challenge this root-cause claim using the supplied issue, logs, and code paths. +Do not edit files. Identify: +1. evidence that contradicts the claim; +2. a simpler competing explanation; +3. a falsification test the supervisor can run; +4. any lifecycle/locking/refcount assumption that is not proven. +Return a concise verdict: supported, uncertain, or contradicted. +``` + +--- + +Next: [04 — Fix planning and implementation](04-fix-and-commits.md) diff --git a/.agents/skills/fix-and-pr/references/04-fix-and-commits.md b/.agents/skills/fix-and-pr/references/04-fix-and-commits.md new file mode 100644 index 00000000..9d8f062f --- /dev/null +++ b/.agents/skills/fix-and-pr/references/04-fix-and-commits.md @@ -0,0 +1,153 @@ +# 04 — Fix planning and implementation + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 4. Phase 3: Implement the fix + +### 4.1 Plan the fix + +Write `build/work/issue-/plan.md`: + +```markdown +# Fix plan: issue + +## Root cause +... + +## Change boundary +Files/functions to change: +- ... + +Explicitly out of scope: +- ... + +## Invariants +- ... + +## Failure paths to preserve +- ... + +## Commit split +1. +2. +3. + +## Verification matrix +| Case | Arch | MODE | SMP | Command | Expected | +| --- | --- | --- | --- | --- | --- | +| Baseline repro | x86_64 | release | 4 | `--expect bug` | 0 | +| Fixed repro | x86_64 | release | 4 | `--expect fixed` | 0 | +| Build | x86_64 | release | 4 | `make build` | 0 | +| Existing smoke | x86_64 | release | 4 | `script/test.sh` | 0 | + +## Risks / open questions +- ... +``` + +For a complex or cross-subsystem fix, have a fresh `reviewer` critique the plan +before any edit: + +```text +Review this fix plan against the established root cause and repository +conventions. Do not edit files. Identify missing lock/lifetime/refcount/error +paths, missing architecture variants, and any step that is broader than the +root cause requires. Return blocking issues first, then non-blocking suggestions. +``` + +Approve the plan only when the change boundary and verification matrix are +explicit. + +### 4.2 Write the fix with one worker + +Use one writer per worktree. The task must state the exact files or subsystem +the worker may touch. + +```text +Objective +Implement the approved fix for issue # according to plan.md. + +Repository and ref +- cwd: +- branch: fix/- +- base commit: + +Authority +- You may edit only: . +- Do not edit unrelated modules, do not reformat files, do not update + dependencies, and do not add debug prints that survive the commit. +- Do not commit unless the supervisor asks you to. +- Stop and report if the plan is wrong or if a second, independent bug is found. + +Required work +1. Read the relevant code and the root-cause analysis before editing. +2. Apply the minimal change that restores the violated invariant. +3. Handle error paths and partial-failure cases. +4. Preserve SMP, interrupt, and preemption safety where applicable. +5. Run `cargo fmt --all -- --check` if the repository expects rustfmt. +6. Build: `make build MODE=release` and `make build MODE=debug` for the target + architecture; add a second architecture when the change is shared. +7. Run the reproduction: + - `--expect bug` must still exit 0 on the baseline commit. + - `--expect fixed` must exit 0 on the fixed tree. +8. Report changed files, commands, exit codes, and log paths. + +Acceptance +- The reproduction passes in fixed mode. +- The baseline still reproduces. +- No unrelated diff. +- No secrets, logs, or build artifacts staged. +``` + +Implementation guardrails: + +- Make the smallest change that fixes the root cause. Do not combine a bug fix + with a broad cleanup. +- If a refactor is required to make the fix safe, keep it in a separate commit. +- Remove temporary instrumentation or put it behind an existing feature flag. +- If the build or reproduction fails, collect evidence and return to the + supervisor. Do not keep editing blindly. +- If a second independent bug appears, record it and ask whether to expand the + scope; do not silently fix it. + +### 4.3 Refine and split the fix + +After the first working version, review the diff as if it were hostile: + +```bash +git diff --stat +git diff +git diff --check +``` + +Check: + +- Is every changed line necessary for the root cause? +- Are lock order, reference ownership, and error paths still correct? +- Is the fix race-free under the reported `SMP` configuration? +- Are architecture-specific assumptions handled? +- Is user input validated on all paths? +- Are there any behavior changes that the issue does not require? +- Are debug prints, temporary hacks, and unrelated formatting removed? + +If the fix is clearer as several commits, split it while preserving +bisectability. A typical sequence is: + +1. `refactor:` or `prep:` — a behavior-preserving preparation, if required. +2. `fix:` — the actual bug fix. +3. `test:` — an optional regression test or reproducer. + +Every commit must build on its own. Use an interactive rebase only on a private +branch: + +```bash +git rebase -i "github/${BASE}" +git diff "github/${BASE}"...HEAD +``` + +Do not rewrite a shared branch that others may have based work on. + +--- + +Next: [05 — Commit conventions](05-commit-conventions.md) diff --git a/.agents/skills/fix-and-pr/references/05-commit-conventions.md b/.agents/skills/fix-and-pr/references/05-commit-conventions.md new file mode 100644 index 00000000..e453aa3e --- /dev/null +++ b/.agents/skills/fix-and-pr/references/05-commit-conventions.md @@ -0,0 +1,70 @@ +# 05 — Commit conventions + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 4.4 Commit conventions + +Follow the repository's existing style. Commit subjects use a subsystem prefix: + +```text +: +``` + +Examples from this repository: + +```text +fix/futex: forget to .await in `futex_exit` +mm: introduce MapFolio for folios mapped to page tables +vfs, inode: remove {read,write}_direct +style, vfs: reformat fat32.rs, file.rs and inode.rs +``` + +Commit body requirements: + +- What failed and under which conditions. +- Why it failed (root cause), not just what changed. +- How the fix restores the invariant. +- Reference the fixed commit or issue: + - `Fixes: ("")` + - `Fixes #` +- Use `Signed-off-by` if the repository expects it (`git commit -s`). +- Keep lines readable (typically 72 columns). + +Example: + +```text +fix/futex: wake waiters in futex_exit + +Testcase `pthread_test` freezes while waiting for clear-tid futex wakeups. +`futex_exit()` forgets to await `do_futex_exit()`, so the exit path returns +before the wakeup is performed. + +Await the exit operation so waiters are woken before the task is reaped. + +Fixes: ("") +Fixes # +Signed-off-by: +``` + +Before committing: + +```bash +git status --short +git diff --cached --name-only +``` + +Commit explicitly, never with `-A`: + +```bash +git add path/to/file.rs path/to/other.rs +git commit -s +``` + +If the commit must be amended after review, re-run the build and reproduction +before force-pushing. + +--- + +Next: [06 — Verification and independent review](06-verification.md) diff --git a/.agents/skills/fix-and-pr/references/06-verification.md b/.agents/skills/fix-and-pr/references/06-verification.md new file mode 100644 index 00000000..afea115b --- /dev/null +++ b/.agents/skills/fix-and-pr/references/06-verification.md @@ -0,0 +1,105 @@ +# 06 — Verification and independent review + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 5. Phase 4: Verification and independent review + +Verification is a separate phase. The author's successful run is not enough. + +### 5.1 Verification matrix + +Run all applicable rows from `plan.md`. At minimum: + +```bash +# Rebuild from the fixed tree. +make build MODE=release ARCH=x86_64 +make build MODE=debug ARCH=x86_64 + +# The exact same reproducer, now expecting the fix. +script/repro/issue-${ISSUE}-${SLUG}.exp \ + --expect fixed --arch x86_64 --mode release --smp 4 \ + --log "$ART/after.log" + +# Existing repository smoke test, if it is applicable and available. +ARCH=x86_64 sh script/test.sh + +# Diff hygiene. +git diff --check +git diff "github/${BASE}"...HEAD +``` + +For shared code, also build and, where feasible, run a second architecture: + +```bash +make build MODE=release ARCH=riscv64 +make build MODE=release ARCH=loongarch64 +``` + +For SMP/race issues: + +- Run the reproduction at least 5–10 times in the fixed configuration. +- Run with `SMP=1` and `SMP=4`. +- Record failures and flakiness, not just the final green run. + +Acceptance for this phase: + +- Baseline `--expect bug` still exits 0 on the base commit. +- Fixed `--expect fixed` exits 0 on the fixed commit. +- Builds pass for the required architectures and modes. +- Existing tests pass, or any failure is proven unrelated and documented. +- No orphan QEMU processes remain. +- Logs are saved and contain no secrets. + +### 5.2 Independent fix review + +Spawn a fresh `reviewer` that did not write the fix: + +```text +Review the diff for issue # against base . Do not edit files. + +Inputs +- diff: `git diff github/...HEAD` +- root cause: build/work/issue-/analysis.md +- plan: build/work/issue-/plan.md +- before/after logs: build/work/issue-/before.log, after.log + +Check, in order: +1. Correctness: does the change actually fix the stated root cause? +2. Lifecycle and ownership: refcounts, locks, lock ordering, awaits, + interrupts, and preemption. +3. Error paths: allocation failure, invalid user pointers, partial setup, + teardown, and double-free/use-after-free. +4. Concurrency: SMP races, lost wakeups, check-then-act windows. +5. Architecture: shared-code assumptions and missing HAL/arch variants. +6. Scope: unrelated changes, debug prints, formatting noise, new dependencies. +7. Verification: does the evidence actually prove the fix, or only a proxy? + +Report blocking findings first, with file:line and a concrete failure scenario. +Then report non-blocking suggestions. If there are no blocking findings, say so +explicitly. Do not approve merely because the reproducer passes. +``` + +If the reviewer finds a blocking issue, return to Phase 3 to correct it, then +re-run the full Phase 4 verification matrix. Do not open a PR with an unresolved +review finding. + +### 5.3 Definition of done + +- [ ] Issue details and expected behavior are recorded. +- [ ] Reproduction script exists and is independently verified. +- [ ] Baseline `--expect bug` exits 0. +- [ ] Fixed `--expect fixed` exits 0. +- [ ] Root cause is stated with evidence. +- [ ] Fix is minimal and scoped to the root cause. +- [ ] Required builds and tests pass. +- [ ] SMP/race behavior is tested when relevant. +- [ ] Independent review has no blocking findings. +- [ ] Commits are clean, build independently, and follow conventions. +- [ ] No secrets, logs, or build artifacts are staged. +- [ ] PR base and head branches are correct. + +--- + +Next: [07 — PR submission](07-pr-submission.md) diff --git a/.agents/skills/fix-and-pr/references/07-pr-submission.md b/.agents/skills/fix-and-pr/references/07-pr-submission.md new file mode 100644 index 00000000..c5afb85e --- /dev/null +++ b/.agents/skills/fix-and-pr/references/07-pr-submission.md @@ -0,0 +1,180 @@ +# 07 — PR submission + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 6. Phase 5: PR submission + +Use a `worker` as publisher. It must not change code during this phase. + +### 6.1 Pre-flight + +```bash +git status --short --branch +git log --oneline "github/${BASE}..HEAD" +git diff --stat "github/${BASE}...HEAD" +git remote -v +``` + +Confirm: + +- The working tree is clean. +- The branch contains only intended commits. +- The base branch is correct. If the issue targets a feature branch, use that + branch as the PR base; do not assume `master`. +- The head branch has been pushed to the repository expected by the PR host. +- No credential material, `GH_TOKEN` value, or workflow logs are in the diff. + +If the base branch is ambiguous, ask the maintainer or supervisor. A PR against +the wrong base can create a misleading diff and break CI. + +### 6.2 Credential handling + +The mandatory gate in [00 — Operating contract](00-operating-contract.md) has +already confirmed that `GH_TOKEN` is present. Re-check it immediately before +publishing: + +```bash +test -n "${GH_TOKEN:-}" || { + echo "GH_TOKEN is not set; ask the user before continuing." >&2 + return 1 +} +``` + +Preferred: use the existing SSH remote for the push and the token only for +GitHub API calls. `GH_TOKEN` is the standard environment variable read by `gh`. +Export it before publishing: + +```bash +gh pr create ... +``` + +Rules: + +- Never `echo "$GH_TOKEN"`, enable `set -x` around credential handling, or + write the value to a file. +- Never embed the token in a remote URL such as + `https://@github.com/...`. +- Never put the token in a subagent prompt, PR body, commit message, or log. +- If an HTTPS push is unavoidable, use a temporary `GIT_ASKPASS` helper that + reads `GH_TOKEN` from the environment, set it to mode `0700`, and remove it + afterward. Prefer `gh`/SSH when available. +- Redact token-bearing URLs from any command output returned to the supervisor. + +### 6.3 Publisher subagent task + +```text +Objective +Publish the verified fix for issue # as a pull request. Do not modify code. + +Parameters +- Local cwd: +- Target repository: greatbridf/osdev +- Base branch: +- Head branch: +- Remote to push to: +- PR title: : +- PR body file: build/work/issue-/pr.md + +Authority +- You may run git push and the GitHub API/CLI. +- Do not edit source files, amend commits, rebase, force-push, or merge. +- Do not print or persist `GH_TOKEN`. +- If authentication fails or the base/head is ambiguous, stop and report. + +Steps +1. Verify the local branch, HEAD sha, clean worktree, and intended diff. +2. Confirm `GH_TOKEN` is present in the environment. Do not read it from a + file. Ensure it is exported; `gh` reads it directly. +3. Push the head branch to its remote. Use SSH if configured; otherwise use a + temporary askpass mechanism that reads `GH_TOKEN`, never a tokenized URL. +4. Create a PR against the target repository with the given base and head. + Include the verification evidence and `Fixes #` in the body. +5. Read back the PR: URL, number, base, head, head SHA, mergeable state, body. +6. Confirm the PR diff matches the local diff and contains no secrets or + unintended files. +7. Report the PR URL, head SHA, checks observed, and any failure. + +Stop conditions +- Push rejected and would require force. +- PR base/head mismatch. +- Token invalid or insufficient scope. +- PR diff contains files outside the intended change. +``` + +Recommended `gh` command: + +```bash +gh pr create \ + --repo greatbridf/osdev \ + --base "$BASE" \ + --head "$HEAD" \ + --title "$TITLE" \ + --body-file "$ART/pr.md" +``` + +If the head branch lives on a fork, use the fully qualified head form required +by the host, for example `--head :`. + +If `gh` is unavailable, use the GitHub REST API with `GH_TOKEN`. Never place the +token in the JSON body or URL. + +### 6.4 PR content + +Use a concise, evidence-based body: + +```markdown +## Summary + + +## Root cause + + +## Fix + + +## Verification +- Baseline reproduction: `script/repro/issue--.exp --expect bug` — exit 0, `before.log` +- Fixed reproduction: `... --expect fixed` — exit 0, `after.log` +- Build: `make build MODE=release ARCH=x86_64` — pass +- Build: `make build MODE=release ARCH=riscv64` — pass (if applicable) +- Existing smoke test: `ARCH=x86_64 sh script/test.sh` — pass (if applicable) +- SMP stress: 10/10 runs passed at `SMP=4` (if applicable) + +## Risks / notes + + +Fixes # +``` + +Do not paste full logs into the PR. Link or summarize them, and attach only what +is necessary. Do not include machine-local absolute paths. + +### 6.5 Post-create validation + +- Open the PR URL and confirm the title, base, head, and body. +- Confirm `Fixes #` is present so the issue closes when merged. +- Confirm CI starts and record the check names. +- Confirm the PR diff matches `git diff "github/${BASE}"...HEAD`. +- Confirm no credential value, `build/`, or workflow logs appear in the diff. +- If CI fails, return to Phase 4 with the failing log; do not "fix" CI by + weakening the reproduction. + +Report to the user: + +```text +PR: +Head: @ +Base: +Reproduction: before=pass (bug), after=pass (fixed) +Builds: +Review: +CI: +``` + +Do not merge the PR unless the user explicitly asks. + +--- + +Next: [08 — Failure handling and escalation](08-failure-handling.md) diff --git a/.agents/skills/fix-and-pr/references/08-failure-handling.md b/.agents/skills/fix-and-pr/references/08-failure-handling.md new file mode 100644 index 00000000..ee69dd40 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/08-failure-handling.md @@ -0,0 +1,25 @@ +# 08 — Failure handling and escalation + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## 7. Failure handling and escalation + +| Situation | Action | +| --- | --- | +| `GH_TOKEN` missing | Stop immediately. Ask the user to provide/export it. Do not start any phase or spawn subagents until it is set. | +| Cannot reproduce | Return to issue intake. Ask for arch, mode, SMP, image, and exact commands. Do not guess a fix. | +| Reproduction is flaky | Run multiple times, record pass rate, distinguish race from infrastructure, and test `SMP=1` vs `SMP=4`. | +| Build fails after a fix | Fix the build first; do not run the reproducer on a stale binary. | +| Fix makes the reproducer pass but root cause is unclear | Stop. A passing proxy is not a root cause. Return to Phase 2. | +| Reviewer finds a blocking issue | Return to Phase 3, correct it, and re-run the full Phase 4 matrix. | +| Fix requires a broad refactor | Split the work or ask for scope approval. Keep the bug-fix PR minimal. | +| Second independent bug appears | Record it, ask whether to expand scope, and do not silently include it. | +| Auth/push/PR failure | Stop and report the exact failure. Never print `GH_TOKEN` or switch to a tokenized URL. | +| Subagent infrastructure failure | Report the run state and partial diff. Do not silently switch execution modes. | +| Base branch ambiguous | Ask. Do not default to `master` when the issue targets a feature branch. | + +--- + +Next: [09 — Issue analysis template](09-analysis-template.md) diff --git a/.agents/skills/fix-and-pr/references/09-analysis-template.md b/.agents/skills/fix-and-pr/references/09-analysis-template.md new file mode 100644 index 00000000..eac2990c --- /dev/null +++ b/.agents/skills/fix-and-pr/references/09-analysis-template.md @@ -0,0 +1,55 @@ +# 09 — Issue analysis template + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## Appendix A: Issue analysis record + +Keep this in `build/work/issue-/analysis.md`: + +```markdown +# Issue : + +## Reported behavior +Expected: +Actual: + +## Environment +- Credential gate: `GH_TOKEN` present (never record the value) +- Commit/branch: +- ARCH: +- MODE: +- SMP: +- QEMU / accelerator: +- Extra image or user program: + +## Reproduction +- Script: +- Command: +- Baseline result: +- Log: + +## Timeline / observations +1. ... +2. ... + +## Candidate hypotheses +| Hypothesis | Evidence for | Evidence against | Test | Result | +| --- | --- | --- | --- | --- | + +## Root cause +<Specific mechanism and evidence.> + +## Fix boundary +Must change: +Must not change: +Invariants: + +## Open questions +- ... +``` + +--- + +Next: [10 — Quick command reference](10-command-reference.md) diff --git a/.agents/skills/fix-and-pr/references/10-command-reference.md b/.agents/skills/fix-and-pr/references/10-command-reference.md new file mode 100644 index 00000000..da40139d --- /dev/null +++ b/.agents/skills/fix-and-pr/references/10-command-reference.md @@ -0,0 +1,46 @@ +# 10 — Quick command reference + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## Appendix B: Quick command reference + +```bash +# Mandatory credential gate +test -n "${GH_TOKEN:-}" || echo "Set GH_TOKEN before doing any work" + +# Environment +./configure +make build MODE=release +make run MODE=release # interactive; exit with Ctrl-A then x +make test-run MODE=release # headless, serial stdio +make srun MODE=release # paused QEMU + GDB stub +make debug # attach GDB +make tmux-debug # both, in tmux +DEBUG_TRAPS=1 make test-run # writes build/qemu.log + +# Reproducer +script/repro/issue-<id>-<slug>.exp --expect bug --log "$ART/before.log" +script/repro/issue-<id>-<slug>.exp --expect fixed --log "$ART/after.log" + +# Diff and commit hygiene +git diff --check +git status --short +git diff --cached --name-only +git log --oneline "github/${BASE}..HEAD" + +# Verification +ARCH=x86_64 sh script/test.sh +make build MODE=release ARCH=riscv64 +make build MODE=release ARCH=loongarch64 + +# PR +gh pr create \ + --repo greatbridf/osdev --base "$BASE" --head "$HEAD" \ + --title "$TITLE" --body-file "$ART/pr.md" +``` + +--- + +Next: [11 — Prompt hygiene](11-prompt-hygiene.md) diff --git a/.agents/skills/fix-and-pr/references/11-prompt-hygiene.md b/.agents/skills/fix-and-pr/references/11-prompt-hygiene.md new file mode 100644 index 00000000..c8a62207 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/11-prompt-hygiene.md @@ -0,0 +1,23 @@ +# 11 — Prompt hygiene + +Part of the GBOS/Eonix `fix-and-pr` skill. Read [00-operating-contract.md](00-operating-contract.md) for the phase gates and hard rules before using this document. + +--- + +## Appendix C: Prompt hygiene + +Every subagent prompt should contain: + +1. **Objective** — one concrete deliverable. +2. **Repository, cwd, and ref** — exact paths and commit/branch. +3. **Authority boundary** — what the subagent may read, edit, run, or publish. +4. **Relevant evidence** — issue, logs, root cause, plan, or diff paths. +5. **Acceptance criteria** — commands and expected exit codes. +6. **Output contract** — changed files, commands run, exit codes, log paths, + verdict, and open questions. +7. **Stop conditions** — ambiguity, missing credentials, infrastructure + failure, or a second independent bug. + +The supervisor checks `GH_TOKEN`; never ask a subagent to discover, retrieve, +or validate the token, and never include the token in a prompt. Prompts must not +contain credentials or unredacted credential-bearing URLs. \ No newline at end of file diff --git a/.agents/skills/fix-and-pr/references/README.md b/.agents/skills/fix-and-pr/references/README.md new file mode 100644 index 00000000..77cfc928 --- /dev/null +++ b/.agents/skills/fix-and-pr/references/README.md @@ -0,0 +1,42 @@ +# Fix-and-PR reference library + +Detailed reference for the GBOS/Eonix `fix-and-pr` skill. The workflow is split +by phase so an agent can load only what it needs. + +Start with [00 — Operating contract](00-operating-contract.md), then read the +document for the phase you are entering. Read the next phase's document before +crossing its gate. + +## Phase documents + +| Phase | Document | Read it when | +| --- | --- | --- | +| 0. Intake and environment | [01 — Environment and issue intake](01-environment-and-intake.md) | Setting up the checkout, build, QEMU, branch, and issue record. | +| 1. Reproduction | [02 — Automated reproduction](02-reproduction.md) | Writing or verifying a headless QEMU reproducer. | +| 2. Analysis | [03 — Root-cause analysis](03-analysis.md) | Collecting evidence, mapping code, and proving the root cause. | +| 3. Fix | [04 — Fix planning and implementation](04-fix-and-commits.md) | Planning the change, writing it, and refining the diff. | +| 3. Commits | [05 — Commit conventions](05-commit-conventions.md) | Splitting and writing commits. | +| 4. Verification | [06 — Verification and review](06-verification.md) | Running the matrix, independent review, and final acceptance. | +| 5. PR | [07 — PR submission](07-pr-submission.md) | Pushing the branch, creating the PR, and validating it. | + +## Supporting documents + +| Document | Purpose | +| --- | --- | +| [08 — Failure handling](08-failure-handling.md) | Stop conditions, escalation, and recovery choices. | +| [09 — Issue analysis template](09-analysis-template.md) | Recording observations, hypotheses, and root cause. | +| [10 — Quick command reference](10-command-reference.md) | Copy-paste commands for build, debug, reproduction, and PR. | +| [11 — Prompt hygiene](11-prompt-hygiene.md) | Required structure for subagent handoffs. | + +## Reading rules + +1. Confirm `GH_TOKEN` is set before doing anything else. If it is missing, stop + and ask the user; do not read further or start a phase. +2. Read [00 — Operating contract](00-operating-contract.md) first. It contains + the roles, hard rules, phase gates, and artifact layout. +3. Load only the current phase document and any supporting document it points + to. +4. Before leaving a phase, read the next phase document and confirm its gate + evidence exists. +5. Do not skip the reproduction or verification documents; they define the + evidence that makes the fix and PR trustworthy. diff --git a/.cargo/config.toml b/.cargo/config.toml index 9c7ba798..12e65525 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,5 +1,4 @@ [build] -target = "riscv64gc-unknown-none-elf" target-dir = 'build' rustflags = ["-C", "force-unwind-tables"] diff --git a/.github/workflows/test-build.yaml b/.github/workflows/test-build.yaml index f6063ab3..f496dae4 100644 --- a/.github/workflows/test-build.yaml +++ b/.github/workflows/test-build.yaml @@ -19,8 +19,10 @@ jobs: include: - arch: riscv64 target: riscv64gc-unknown-none-elf - - arch: loongarch64 - target: loongarch64-unknown-none-softfloat + platform: virt + - arch: x86_64 + target: x86_64-unknown-none + platform: acpi_compat steps: - name: Checkout code uses: actions/checkout@v4 @@ -30,7 +32,8 @@ jobs: - name: Setup QEMU run: | - sudo apt-get install -y qemu-system-${{ matrix.arch }} qemu-kvm + sudo apt-get update + sudo apt-get install -y qemu-system-${{ matrix.arch }} qemu-kvm expect - name: Configure run: ./configure @@ -38,7 +41,7 @@ jobs: - name: Run build for ${{ matrix.arch }} targets run: | make build ARCH=${{ matrix.arch }} MODE=release - zstd -k build/boot-${{ matrix.arch }}.img + zstd -k build/boot-${{ matrix.arch }}-${{ matrix.platform }}.img - name: Upload build artifacts uses: actions/upload-artifact@v4 @@ -46,7 +49,7 @@ jobs: name: eonix-kernel-and-image-${{ matrix.arch }} path: | build/${{ matrix.target }}/release/eonix_kernel - build/boot-${{ matrix.arch }}.img.zst + build/boot-${{ matrix.arch }}-${{ matrix.platform }}.img.zst - name: Test run for ${{ matrix.arch }} run: | diff --git a/.gitignore b/.gitignore index fbc2a9b1..4684b698 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ build/ .idea/ +.vscode/settings.json test/ diff --git a/.rustfmt.toml b/.rustfmt.toml index d69872c6..1075fd32 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1,4 +1,4 @@ -max_width = 100 +max_width = 80 hard_tabs = false tab_spaces = 4 newline_style = "Auto" @@ -14,7 +14,7 @@ single_line_if_else_max_width = 60 single_line_let_else_max_width = 60 wrap_comments = false format_code_in_doc_comments = false -doc_comment_code_block_width = 100 +doc_comment_code_block_width = 80 comment_width = 80 normalize_comments = false normalize_doc_attributes = false @@ -29,8 +29,8 @@ fn_single_line = false where_single_line = false imports_indent = "Block" imports_layout = "Mixed" -imports_granularity = "Preserve" -group_imports = "Preserve" +imports_granularity = "Module" +group_imports = "StdExternalCrate" reorder_imports = true reorder_modules = true reorder_impl_items = false @@ -42,13 +42,13 @@ binop_separator = "Front" remove_nested_parens = true combine_control_expr = true short_array_element_width_threshold = 10 -overflow_delimited_expr = false +overflow_delimited_expr = true struct_field_align_threshold = 0 enum_discrim_align_threshold = 0 match_arm_blocks = true match_arm_leading_pipes = "Never" force_multiline_blocks = false -fn_params_layout = "Tall" +fn_params_layout = "Compressed" brace_style = "SameLineWhere" control_brace_style = "AlwaysSameLine" trailing_semicolon = true diff --git a/.vscode/settings.example.json b/.vscode/settings.example.json new file mode 100644 index 00000000..fbafa867 --- /dev/null +++ b/.vscode/settings.example.json @@ -0,0 +1,10 @@ +{ + "editor.formatOnSave": true, + // Uncomment and set the target you'd like to use for analysis. + // + // Available targets: + // - x86_64-unknown-none.json + // - riscv64gc-unknown-none-elf + // - loongarch64-unknown-none-softfloat + // "rust-analyzer.cargo.target": "x86_64-unknown-none.json", +} diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 634d16af..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "makefile.configureOnOpen": false, -} diff --git a/Cargo.lock b/Cargo.lock index 59242bbc..896ec493 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,21 @@ dependencies = [ "log", ] +[[package]] +name = "arcref" +version = "0.1.0" + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic_unique_refcell" version = "0.1.0" @@ -55,14 +70,13 @@ name = "buddy_allocator" version = "0.1.0" dependencies = [ "eonix_mm", - "intrusive_list", ] [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "critical-section" @@ -144,9 +158,12 @@ dependencies = [ "acpi", "align_ext", "another_ext4", + "arcref", + "async-trait", "atomic_unique_refcell", "bitflags", "buddy_allocator", + "cfg-if", "eonix_hal", "eonix_log", "eonix_macros", @@ -155,6 +172,7 @@ dependencies = [ "eonix_preempt", "eonix_runtime", "eonix_sync", + "futures", "intrusive-collections 0.9.8", "intrusive_list", "itertools", @@ -162,6 +180,7 @@ dependencies = [ "posix_types", "slab_allocator", "stalloc", + "static_assertions", "unwinding", "virtio-drivers", "xmas-elf", @@ -265,6 +284,79 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784a4df722dc6267a04af36895398f59d21d07dce47232adf31ec0ff2fa45e67" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-macro", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", +] + [[package]] name = "gimli" version = "0.32.0" @@ -332,6 +424,18 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pointers" version = "0.1.0" @@ -413,7 +517,6 @@ version = "0.1.0" dependencies = [ "eonix_mm", "eonix_sync", - "intrusive_list", ] [[package]] @@ -422,6 +525,12 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a37f0ead4094eeb54c6893316aa139e48b252f1c07511e5124fa1f9414df5b6c" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "syn" version = "2.0.104" diff --git a/Cargo.toml b/Cargo.toml index e70d8c65..f5df1fa1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,9 @@ edition = "2021" crate-type = ["bin"] [dependencies] +arcref = { path = "./crates/arcref", default-features = false, features = [ + "alloc", +] } atomic_unique_refcell = { path = "./crates/atomic_unique_refcell", features = [ "no_std", ] } @@ -24,8 +27,10 @@ pointers = { path = "./crates/pointers" } posix_types = { path = "./crates/posix_types" } slab_allocator = { path = "./crates/slab_allocator" } +intrusive-collections = { version = "0.9.8", features = [ + "nightly", +], git = "https://github.com/greatbridf/intrusive-rs" } bitflags = "2.6.0" -intrusive-collections = { version = "0.9.8", git = "https://github.com/greatbridf/intrusive-rs" } itertools = { version = "0.13.0", default-features = false } acpi = "5.2.0" align_ext = "0.1.0" @@ -34,11 +39,24 @@ another_ext4 = { git = "https://github.com/SMS-Derfflinger/another_ext4", branch stalloc = { version = "0.6.1", default-features = false, features = [ "allocator-api", ] } +async-trait = "0.1.89" +futures = { version = "0.3.31", features = [ + "alloc", + "async-await", +], default-features = false } +static_assertions = "1.1.0" +cfg-if = "1.0.4" -[target.'cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))'.dependencies] -virtio-drivers = { version = "0.11.0" } +[target.'cfg(target_arch = "x86_64")'.dependencies] +unwinding = { version = "0.2.8", default-features = false, features = [ + "unwinder", + "fde-static", + "personality", + "panic", +] } [target.'cfg(target_arch = "riscv64")'.dependencies] +virtio-drivers = { version = "0.11.0" } unwinding = { version = "0.2.8", default-features = false, features = [ "unwinder", "fde-static", @@ -46,6 +64,9 @@ unwinding = { version = "0.2.8", default-features = false, features = [ "panic", ] } +[target.'cfg(target_arch = "loongarch64")'.dependencies] +virtio-drivers = { version = "0.11.0" } + [features] default = [] trace_pci = [] diff --git a/Makefile.src b/Makefile.src index ab13f5b8..eca1eda4 100644 --- a/Makefile.src +++ b/Makefile.src @@ -54,6 +54,8 @@ endif ifeq ($(ARCH),riscv64) +PLATFORM ?= virt + BINARY_DIR_BASE := build/riscv64gc-unknown-none-elf BINARY_DIR := $(BINARY_DIR_BASE)/$(MODE) @@ -61,7 +63,7 @@ QEMU_ARGS += \ -machine virt -kernel $(BINARY_DIR)/eonix_kernel \ -device virtio-blk-device,drive=disk0,bus=virtio-mmio-bus.0 \ -device virtio-net-device,netdev=mynet0 \ - -drive id=disk0,file=build/boot-riscv64.img,format=raw,if=none \ + -drive id=disk0,file=build/boot-riscv64-virt.img,format=raw,if=none \ -netdev user,id=mynet0 \ -rtc base=utc @@ -73,11 +75,10 @@ endif CARGO_FLAGS += --target riscv64gc-unknown-none-elf -.PHONY: build -build: $(BINARY_DIR)/eonix_kernel build/boot-riscv64.img - else ifeq ($(ARCH),loongarch64) +PLATFORM ?= virt + BINARY_DIR_BASE := build/loongarch64-unknown-none-softfloat BINARY_DIR := $(BINARY_DIR_BASE)/$(MODE) @@ -85,7 +86,7 @@ QEMU_ARGS += \ -machine virt -kernel $(BINARY_DIR)/eonix_kernel -m 1G \ -device virtio-blk-pci,drive=disk0 \ -device virtio-net-pci,netdev=mynet0 \ - -drive id=disk0,file=build/boot-loongarch64.img,format=raw,if=none \ + -drive id=disk0,file=build/boot-loongarch64-virt.img,format=raw,if=none \ -netdev user,id=mynet0,hostfwd=tcp::5555-:5555,hostfwd=udp::5555-:5555 \ -rtc base=utc @@ -97,11 +98,10 @@ endif CARGO_FLAGS += --target loongarch64-unknown-none-softfloat -.PHONY: build -build: $(BINARY_DIR)/eonix_kernel build/boot-loongarch64.img - else ifeq ($(ARCH),x86_64) +PLATFORM ?= acpi_compat + BINARY_DIR_BASE := build/x86_64-unknown-none BINARY_DIR := $(BINARY_DIR_BASE)/$(MODE) @@ -110,7 +110,7 @@ QEMU_ARGS += \ -device ahci,id=ahci \ -device ide-hd,drive=disk0,bus=ahci.0 \ -device e1000e,netdev=mynet0 \ - -drive id=disk0,file=build/boot-x86_64.img,format=raw,if=none \ + -drive id=disk0,file=build/boot-x86_64-acpi_compat.img,format=raw,if=none \ -netdev user,id=mynet0 ifneq ($(IMG),) @@ -121,11 +121,13 @@ endif CARGO_FLAGS += --target x86_64-unknown-none.json -.PHONY: build -build: $(BINARY_DIR)/eonix_kernel build/boot-x86_64.img - endif +TARGET := $(ARCH)-$(PLATFORM) + +.PHONY: build +build: $(BINARY_DIR)/eonix_kernel build/boot-$(TARGET).img + .PHONY: run run: build build/kernel.sym $(QEMU) $(QEMU_ARGS) -display none -serial mon:stdio @@ -173,8 +175,8 @@ $(BINARY_DIR)/eonix_kernel: $(KERNEL_DEPS) build/kernel.sym: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- --only-keep-debug build/kernel.sym -build/fs-%.img: user-programs/init_script_%.sh script/build-img.sh $(USER_PROGRAMS) - ARCH=$* OUTPUT=$@ sh script/build-img.sh +build/fs-%.img: user-programs/init_script_%.sh script/build-img/build-img $(USER_PROGRAMS) + script/build-img/build-img -t $* -o $@ build/mbr.bin: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- -O binary -j .mbr build/mbr.bin @@ -186,7 +188,7 @@ build/kernel.bin: $(BINARY_DIR)/eonix_kernel CARGO_TARGET_DIR=build cargo objcopy -q $(CARGO_FLAGS) -- -O binary --strip-debug \ -R .mbr -R .stage1 build/kernel.bin -build/boot-x86_64.img: build/fs-x86_64.img build/mbr.bin build/stage1.bin build/kernel.bin +build/boot-x86_64-acpi_compat.img: build/fs-x86_64-acpi_compat.img build/mbr.bin build/stage1.bin build/kernel.bin dd if=build/mbr.bin of=$@ bs=512 count=1 conv=notrunc 2> /dev/null dd if=build/stage1.bin of=$@ bs=512 seek=1 conv=notrunc 2> /dev/null dd if=build/kernel.bin of=$@ bs=4096 seek=1 conv=notrunc 2> /dev/null @@ -195,13 +197,13 @@ build/boot-x86_64.img: build/fs-x86_64.img build/mbr.bin build/stage1.bin build/ sh -c 'echo n; echo; echo; echo 8192; echo; echo a; echo w' \ | $(FDISK) $@ 2> /dev/null > /dev/null -build/boot-riscv64.img: build/fs-riscv64.img +build/boot-riscv64-virt.img: build/fs-riscv64-virt.img dd if=$< of=$@ bs=$(shell expr 4 \* 1024 \* 1024) \ seek=1 conv=notrunc 2> /dev/null sh -c 'echo n; echo; echo; echo 8192; echo; echo a; echo w' \ | $(FDISK) $@ 2> /dev/null > /dev/null -build/boot-loongarch64.img: build/fs-loongarch64.img +build/boot-loongarch64-virt.img: build/fs-loongarch64-virt.img dd if=$< of=$@ bs=$(shell expr 4 \* 1024 \* 1024) \ seek=1 conv=notrunc 2> /dev/null sh -c 'echo n; echo; echo; echo 8192; echo; echo a; echo w' \ diff --git a/build.rs b/build.rs index 1c9d2c3d..cea0d5d6 100644 --- a/build.rs +++ b/build.rs @@ -1,6 +1,70 @@ +const ARCH_CONFIGS: &[&str] = &["has_stacktrace", "has_shutdown"]; + +// Define configs + +fn define_config(config: &str) { + println!("cargo::rustc-check-cfg=cfg({config})"); +} + +fn define_arch_config(config: &str) { + define_config(&format!("arch_{config}")); +} + +fn define_arch_configs() { + for config in ARCH_CONFIGS { + define_arch_config(config); + } +} + +fn define_configs() { + define_arch_configs(); +} + +// Set configs + +fn set_config(config: &str) { + println!("cargo::rustc-cfg={config}"); +} + +fn set_arch_config(config: &str) { + set_config(&format!("arch_{config}")); +} + +fn set_arch_configs_x86() { + set_arch_config("has_stacktrace"); + set_arch_config("has_shutdown"); +} + +fn set_arch_configs_riscv64() { + set_arch_config("has_stacktrace"); + set_arch_config("has_shutdown"); +} + +fn set_arch_configs_loongarch64() { + set_arch_config("has_shutdown"); +} + +fn set_arch_configs() { + match std::env::var("CARGO_CFG_TARGET_ARCH").as_deref().unwrap() { + "x86_64" => set_arch_configs_x86(), + "riscv64" => set_arch_configs_riscv64(), + "loongarch64" => set_arch_configs_loongarch64(), + arch => panic!("Unsupported architecture: {}", arch), + } +} + +fn set_configs() { + set_arch_configs(); +} + fn main() -> Result<(), Box<dyn std::error::Error>> { + define_configs(); + set_configs(); + println!("cargo:rustc-link-arg=-T{}", "link.x"); - if let Ok(extra_link_args) = std::env::var("DEP_EONIX_HAL_EXTRA_LINK_ARGS") { + + if let Ok(extra_link_args) = std::env::var("DEP_EONIX_HAL_EXTRA_LINK_ARGS") + { for arg in extra_link_args.split_whitespace() { println!("cargo:rustc-link-arg={}", arg); } diff --git a/crates/arcref/Cargo.lock b/crates/arcref/Cargo.lock new file mode 100644 index 00000000..3c4e1567 --- /dev/null +++ b/crates/arcref/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arcref" +version = "0.1.0" diff --git a/crates/arcref/Cargo.toml b/crates/arcref/Cargo.toml new file mode 100644 index 00000000..a0af89f8 --- /dev/null +++ b/crates/arcref/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "arcref" +version = "0.1.0" +edition = "2024" + +[dependencies] + +[features] +alloc = [] +std = ["alloc"] +default = ["std"] diff --git a/crates/arcref/src/arcref.rs b/crates/arcref/src/arcref.rs new file mode 100644 index 00000000..3d01852d --- /dev/null +++ b/crates/arcref/src/arcref.rs @@ -0,0 +1,216 @@ +#[cfg(not(feature = "std"))] +use core::{ + borrow::Borrow, + marker::{PhantomData, Unsize}, + mem::ManuallyDrop, + ops::{Deref, DispatchFromDyn}, +}; + +#[cfg(all(not(feature = "std"), feature = "alloc"))] +extern crate alloc; + +#[cfg(all(not(feature = "std"), feature = "alloc"))] +use alloc::sync::Arc; + +#[cfg(feature = "std")] +use std::{ + borrow::Borrow, + marker::{PhantomData, Unsize}, + mem::ManuallyDrop, + ops::{Deref, DispatchFromDyn}, + sync::Arc, +}; + +pub trait AsArcRef<T> +where + T: ?Sized, +{ + /// Borrow the [`Arc`] and convert the reference into [`ArcRef`]. + fn aref(&self) -> ArcRef<'_, T>; +} + +pub struct ArcRef<'a, T: ?Sized> { + ptr: *const T, + _phantom: PhantomData<&'a ()>, +} + +unsafe impl<T: ?Sized + Send + Sync> Send for ArcRef<'_, T> {} +unsafe impl<T: ?Sized + Send + Sync> Sync for ArcRef<'_, T> {} + +#[cfg(any(feature = "std", feature = "alloc"))] +impl<'a, T: ?Sized> ArcRef<'a, T> { + pub fn new(arc: &'a Arc<T>) -> Self { + Self { + ptr: Arc::as_ptr(arc), + _phantom: PhantomData, + } + } + + /// Create a new `ArcRef` from a raw pointer. + /// + /// # Safety + /// The given pointer MUST be created by `Arc::as_ptr` or `Arc::into_raw`. + /// The caller is responsible to ensure that the pointer is valid for the + /// lifetime of the `ArcRef`. + pub unsafe fn new_unchecked(arc_ptr: *const T) -> Self { + Self { + ptr: arc_ptr, + _phantom: PhantomData, + } + } + + pub fn with_arc<Func, Out>(self, func: Func) -> Out + where + Func: FnOnce(&Arc<T>) -> Out, + { + func(&ManuallyDrop::new(unsafe { Arc::from_raw(self.ptr) })) + } + + pub fn clone_arc(self) -> Arc<T> { + self.with_arc(|arc| arc.clone()) + } + + pub fn ptr_eq_arc(self, other: &Arc<T>) -> bool { + self.with_arc(|arc| Arc::ptr_eq(arc, other)) + } +} + +#[cfg(all(not(feature = "std"), feature = "alloc"))] +impl<T> AsArcRef<T> for Arc<T> +where + T: ?Sized, +{ + fn aref(&self) -> ArcRef<'_, T> { + ArcRef::new(self) + } +} + +impl<T> AsRef<T> for ArcRef<'_, T> +where + T: ?Sized, +{ + fn as_ref(&self) -> &T { + self.deref() + } +} + +impl<T> Borrow<T> for ArcRef<'_, T> +where + T: ?Sized, +{ + fn borrow(&self) -> &T { + self.deref() + } +} + +impl<'a, T> Clone for ArcRef<'a, T> +where + T: ?Sized, +{ + fn clone(&self) -> Self { + Self { + ptr: self.ptr, + _phantom: PhantomData, + } + } +} + +impl<T> Copy for ArcRef<'_, T> where T: ?Sized {} + +impl<T: ?Sized> Deref for ArcRef<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + unsafe { + // SAFETY: `self.ptr` points to a valid `T` instance because it was + // created from a valid `Arc<T>`. + self.ptr.as_ref().unwrap_unchecked() + } + } +} + +impl<'a, T, U> DispatchFromDyn<ArcRef<'a, U>> for ArcRef<'a, T> +where + T: ?Sized + Unsize<U>, + U: ?Sized, +{ +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn create_from_arc() { + let data = Arc::new(42); + let _arc_ref = ArcRef::new(&data); + } + + #[test] + fn deref() { + let data = Arc::new(42); + let arc_ref = ArcRef::new(&data); + + assert_eq!(*arc_ref, 42); + } + + #[test] + fn clone_into_arc() { + let data = Arc::new(42); + let arc_ref = ArcRef::new(&data); + + let cloned = arc_ref.clone_arc(); + + assert_eq!(Arc::strong_count(&data), 2); + assert_eq!(*cloned, 42); + } + + #[test] + fn dyn_compatible_receiver() { + struct Data(u32); + + trait Trait { + fn foo(self: ArcRef<Self>) -> u32; + } + + impl Trait for Data { + fn foo(self: ArcRef<Self>) -> u32 { + self.0 + } + } + + let data = Arc::new(Data(42)); + let arc_ref = ArcRef::new(&data); + + assert_eq!(arc_ref.foo(), 42); + } + + #[test] + fn clone_from_train_methods() { + struct Data(u32); + + trait Trait { + fn foo(&self) -> u32; + + fn clone_self(self: ArcRef<Self>) -> Arc<dyn Trait>; + } + + impl Trait for Data { + fn foo(&self) -> u32 { + self.0 + } + + fn clone_self(self: ArcRef<Self>) -> Arc<dyn Trait> { + self.clone_arc() as _ + } + } + + let data = Arc::new(Data(42)); + let arc_ref = ArcRef::new(&data); + + let cloned = arc_ref.clone_self(); + + assert_eq!(arc_ref.foo(), 42); + assert_eq!(cloned.foo(), 42); + } +} diff --git a/crates/arcref/src/lib.rs b/crates/arcref/src/lib.rs new file mode 100644 index 00000000..83a61985 --- /dev/null +++ b/crates/arcref/src/lib.rs @@ -0,0 +1,8 @@ +#![cfg_attr(not(feature = "std"), no_std)] +#![feature(arbitrary_self_types)] +#![feature(dispatch_from_dyn)] +#![feature(unsize)] + +mod arcref; + +pub use arcref::{ArcRef, AsArcRef}; diff --git a/crates/buddy_allocator/Cargo.toml b/crates/buddy_allocator/Cargo.toml index 51f02295..bdb0a28c 100644 --- a/crates/buddy_allocator/Cargo.toml +++ b/crates/buddy_allocator/Cargo.toml @@ -5,4 +5,3 @@ edition = "2024" [dependencies] eonix_mm = { path = "../eonix_mm" } -intrusive_list = { path = "../intrusive_list" } diff --git a/crates/buddy_allocator/src/debug.rs b/crates/buddy_allocator/src/debug.rs new file mode 100644 index 00000000..3942c67d --- /dev/null +++ b/crates/buddy_allocator/src/debug.rs @@ -0,0 +1,36 @@ +use eonix_mm::paging::{FolioList, Zone}; + +use crate::{BuddyAllocator, BuddyFolio, AREAS}; + +#[derive(Clone, Copy)] +pub struct BuddyStat { + pub alloced_count: usize, + pub free_count: usize, + pub order: u32, +} + +impl<Z, L, F> BuddyAllocator<Z, L> +where + Z: Zone<Page = F>, + L: FolioList<Folio = F>, + F: BuddyFolio + 'static, +{ + pub fn dump_stat(&self) -> [BuddyStat; AREAS] { + let mut stats = [BuddyStat { + alloced_count: 0, + free_count: 0, + order: 0, + }; AREAS]; + + for i in 0..AREAS { + let area = &self.free_areas[i]; + let stat = &mut stats[i]; + + stat.alloced_count = area.alloced; + stat.free_count = area.count; + stat.order = i as u32; + } + + stats + } +} diff --git a/crates/buddy_allocator/src/free_area.rs b/crates/buddy_allocator/src/free_area.rs deleted file mode 100644 index 837f733f..00000000 --- a/crates/buddy_allocator/src/free_area.rs +++ /dev/null @@ -1,59 +0,0 @@ -use crate::BuddyRawPage; -use core::marker::{PhantomData, Send, Sync}; -use intrusive_list::Link; - -pub struct FreeArea<T> { - free_list: Link, - count: usize, - _phantom: PhantomData<T>, -} - -unsafe impl<T> Send for FreeArea<T> {} -unsafe impl<T> Sync for FreeArea<T> {} - -impl<Raw> FreeArea<Raw> -where - Raw: BuddyRawPage, -{ - pub const fn new() -> Self { - Self { - free_list: Link::new(), - count: 0, - _phantom: PhantomData, - } - } - - pub fn get_free_pages(&mut self) -> Option<Raw> { - self.free_list.next_mut().map(|pages_link| { - assert_ne!(self.count, 0); - - let pages_ptr = unsafe { - // SAFETY: Items in `self.free_list` are guaranteed to be of type `Raw`. - Raw::from_link(pages_link) - }; - - self.count -= 1; - pages_link.remove(); - - pages_ptr - }) - } - - pub fn add_pages(&mut self, pages_ptr: Raw) { - self.count += 1; - pages_ptr.set_free(); - - unsafe { - self.free_list.insert(pages_ptr.get_link()); - } - } - - pub fn del_pages(&mut self, pages_ptr: Raw) { - assert!(self.count >= 1 && pages_ptr.is_free()); - self.count -= 1; - pages_ptr.clear_free(); - unsafe { - pages_ptr.get_link().remove(); - } - } -} diff --git a/crates/buddy_allocator/src/lib.rs b/crates/buddy_allocator/src/lib.rs index f8c8eeda..c4dfa7c9 100644 --- a/crates/buddy_allocator/src/lib.rs +++ b/crates/buddy_allocator/src/lib.rs @@ -1,87 +1,262 @@ #![no_std] -mod free_area; -mod zone; +pub mod debug; -use core::sync::atomic::Ordering; -use eonix_mm::{ - address::PAddr, - paging::{RawPage, PFN}, -}; -use intrusive_list::Link; -use zone::Zone; +use core::hint::unreachable_unchecked; + +use eonix_mm::address::{AddrOps as _, PAddr, PRange}; +use eonix_mm::paging::{FolioList, FolioListSized, Zone, PFN}; const MAX_ORDER: u32 = 10; -const ZONE_AREAS: usize = const { MAX_ORDER as usize + 1 }; +const AREAS: usize = const { MAX_ORDER as usize + 1 }; -pub trait BuddyRawPage: RawPage { - /// Get the container raw page struct of the list link. - /// - /// # Safety - /// The caller MUST ensure that the link points to a `RawPage`. - unsafe fn from_link(link: &mut Link) -> Self; +pub trait BuddyFolio: Sized + 'static { + fn pfn(&self) -> PFN; - /// Get the list link of the raw page. - /// - /// # Safety - /// The caller MUST ensure that at any time, only one mutable reference - /// to the link exists. - unsafe fn get_link(&self) -> &mut Link; + fn get_order(&self) -> u32; + fn is_buddy(&self) -> bool; - fn set_order(&self, order: u32); + fn set_order(&mut self, order: u32); + fn set_buddy(&mut self, value: bool); +} - fn is_buddy(&self) -> bool; - fn is_free(&self) -> bool; +struct FreeArea<L> +where + L: FolioList, +{ + free_list: L, - fn set_buddy(&self); - fn set_free(&self); + alloced: usize, + count: usize, +} - fn clear_buddy(&self); - fn clear_free(&self); +unsafe impl<L> Send for FreeArea<L> where L: FolioList {} +unsafe impl<L> Sync for FreeArea<L> where L: FolioList {} + +pub struct BuddyAllocator<Z, L> +where + Z: Zone + 'static, + L: FolioList, +{ + zone: &'static Z, + free_areas: [FreeArea<L>; AREAS], } -pub struct BuddyAllocator<T> +impl<Z, L> BuddyAllocator<Z, L> where - T: BuddyRawPage, + Z: Zone + 'static, + Z::Page: BuddyFolio, + L: FolioListSized, { - zone: Zone<T, ZONE_AREAS>, + pub const fn new(zone: &'static Z) -> Self { + Self { + zone, + free_areas: [const { FreeArea::new() }; AREAS], + } + } } -impl<T> BuddyAllocator<T> +impl<Z, L, F> BuddyAllocator<Z, L> where - T: BuddyRawPage, + Z: Zone<Page = F>, + L: FolioList<Folio = F>, + F: BuddyFolio + 'static, { - pub const fn new() -> Self { - Self { zone: Zone::new() } + pub fn create_folios(&mut self, start: PAddr, end: PAddr) { + assert!( + self.zone + .contains_prange(PRange::new(start.ceil(), end.floor())), + "The given address range is not within the zone." + ); + + let mut pfn = PFN::from(start.ceil()); + let end_pfn = PFN::from(end.floor()); + + while pfn < end_pfn { + let mut order = usize::from(pfn).trailing_zeros().min(MAX_ORDER); + let new_end_pfn = loop { + let new_end = pfn + (1 << order); + + if new_end <= end_pfn { + break new_end; + } + + order -= 1; + }; + + unsafe { + // SAFETY: We've checked that the range is within the zone above. + self.add_folio_unchecked(pfn, order) + }; + + pfn = new_end_pfn; + } + } + + fn add_folio(&mut self, pfn: PFN, order: u32) { + let prange = PRange::from(PAddr::from(pfn)).grow(1 << (order + 12)); + assert!( + self.zone.contains_prange(prange), + "The given folio is not within the zone." + ); + + unsafe { + // SAFETY: Checks above. + self.add_folio_unchecked(pfn, order); + } + } + + unsafe fn add_folio_unchecked(&mut self, pfn: PFN, order: u32) { + let Some(mut folio) = self.zone.get_page(pfn) else { + unsafe { unreachable_unchecked() } + }; + + unsafe { + // SAFETY: The caller ensures that the page is unused. + let folio_mut = folio.as_mut(); + self.free_areas[order as usize].add_folio(folio_mut, order); + } + } + + fn break_folio(&mut self, folio: &mut F, order: u32, target_order: u32) { + let pfn = folio.pfn(); + + for order in (target_order..order).rev() { + let buddy_pfn = pfn + (1 << order); + + unsafe { + // SAFETY: We got the page from `self.free_areas`. Checks are + // done when we've put the page into the buddy system. + self.add_folio_unchecked(buddy_pfn, order); + } + } + + folio.set_order(target_order); } - pub fn create_pages(&mut self, start: PAddr, end: PAddr) { - self.zone.create_pages(start, end); + pub fn alloc_order(&mut self, order: u32) -> Option<&'static mut Z::Page> { + for current_order in order..AREAS as u32 { + let area = &mut self.free_areas[current_order as usize]; + + let Some(folio) = area.get_free_folio() else { + continue; + }; + + if current_order > order { + self.break_folio(folio, current_order, order); + } + + self.free_areas[order as usize].alloced += 1; + return Some(folio); + } + + None } - pub fn alloc_order(&mut self, order: u32) -> Option<T> { - let pages_ptr = self.zone.get_free_pages(order); + pub unsafe fn dealloc(&mut self, folio: &'static mut Z::Page) { + let mut pfn = folio.pfn(); + let mut order = folio.get_order(); + + assert!( + !folio.is_buddy(), + "Trying to free a folio that is already in the buddy system: {pfn:?}", + ); + + self.free_areas[order as usize].alloced -= 1; - if let Some(pages_ptr) = pages_ptr { - // SAFETY: Memory order here can be Relaxed is for the same reason as that - // in the copy constructor of `std::shared_ptr`. - pages_ptr.refcount().fetch_add(1, Ordering::Relaxed); - pages_ptr.clear_free(); + while order < MAX_ORDER { + let buddy_pfn = pfn.buddy_pfn(order); + let Some(buddy) = self.try_get_buddy(buddy_pfn, order) else { + break; + }; + + self.free_areas[order as usize].remove_folio(buddy); + pfn = pfn.combined_pfn(buddy_pfn); + order += 1; } - pages_ptr + self.add_folio(pfn, order); } - pub unsafe fn dealloc(&mut self, page_ptr: T) { - self.zone.free_pages(page_ptr); + /// This function checks whether the given page is within our [`Zone`] and + /// is a free buddy page with the specified order. + /// + /// We can assure exclusive access to a buddy page of [`order`] if + /// - the buddy is within the same [`Zone`] as us. + /// - the buddy is a free buddy (in some [`FreeArea`]) + /// - the buddy has order [`order`] + fn try_get_buddy<'a>( + &mut self, buddy_pfn: PFN, order: u32, + ) -> Option<&'a mut F> { + let mut buddy = self.zone.get_page(buddy_pfn)?; + + unsafe { + // SAFETY: We just test whether the page is a buddy. + let buddy_ref = buddy.as_ref(); + + if !buddy_ref.is_buddy() { + return None; + } + + // Sad... + if buddy_ref.get_order() != order { + return None; + } + + // SAFETY: We have the mutable reference to the buddy allocator. + // So all the pages within are exclusively accessible to us. + Some(buddy.as_mut()) + } } +} + +impl<L> FreeArea<L> +where + L: FolioListSized, +{ + const fn new() -> Self { + Self { + free_list: L::NEW, + alloced: 0, + count: 0, + } + } +} + +impl<L> FreeArea<L> +where + L: FolioList, + L::Folio: BuddyFolio + 'static, +{ + pub fn get_free_folio(&mut self) -> Option<&'static mut L::Folio> { + self.free_list.pop_head().map(|folio| { + assert_ne!(self.count, 0, "Oops"); + + folio.set_buddy(false); + self.count -= 1; + + folio + }) + } + + pub fn add_folio(&mut self, folio: &'static mut L::Folio, order: u32) { + folio.set_order(order); + folio.set_buddy(true); + + self.count += 1; + self.free_list.push_tail(folio); + } + + pub fn remove_folio(&mut self, folio: &mut L::Folio) { + assert_ne!(self.count, 0, "Oops"); + folio.set_buddy(false); - pub fn has_management_over(page_ptr: T) -> bool { - !page_ptr.is_free() && page_ptr.is_buddy() + self.count -= 1; + self.free_list.remove(folio); } } -pub(self) trait BuddyPFNOps { +trait BuddyPFNOps { fn buddy_pfn(self, order: u32) -> PFN; fn combined_pfn(self, buddy_pfn: PFN) -> PFN; } diff --git a/crates/buddy_allocator/src/zone.rs b/crates/buddy_allocator/src/zone.rs deleted file mode 100644 index 2c850ef9..00000000 --- a/crates/buddy_allocator/src/zone.rs +++ /dev/null @@ -1,146 +0,0 @@ -use super::free_area::FreeArea; -use crate::{BuddyPFNOps as _, BuddyRawPage}; -use core::sync::atomic::Ordering; -use eonix_mm::{ - address::{AddrOps as _, PAddr}, - paging::PFN, -}; - -pub(super) struct Zone<T, const AREAS: usize> { - free_areas: [FreeArea<T>; AREAS], -} - -impl<Raw, const AREAS: usize> Zone<Raw, AREAS> -where - Raw: BuddyRawPage, -{ - pub const fn new() -> Self { - Self { - free_areas: [const { FreeArea::new() }; AREAS], - } - } - - pub fn get_free_pages(&mut self, order: u32) -> Option<Raw> { - for current_order in order..AREAS as u32 { - let pages_ptr = self.free_areas[current_order as usize].get_free_pages(); - let Some(pages_ptr) = pages_ptr else { continue }; - - pages_ptr.set_order(order); - - if current_order > order { - self.expand(pages_ptr, current_order, order); - } - - assert!( - pages_ptr.is_present(), - "Page {:?} is not present", - pages_ptr.into(), - ); - - assert!( - pages_ptr.is_free(), - "Page {:?} is not free", - pages_ptr.into(), - ); - - return Some(pages_ptr); - } - None - } - - fn expand(&mut self, pages_ptr: Raw, order: u32, target_order: u32) { - let mut offset = 1 << order; - let pages_pfn = Into::<PFN>::into(pages_ptr); - - for order in (target_order..order).rev() { - offset >>= 1; - - let split_pages_ptr = Raw::from(pages_pfn + offset); - split_pages_ptr.set_order(order); - split_pages_ptr.set_buddy(); - self.free_areas[order as usize].add_pages(split_pages_ptr); - } - } - - pub fn free_pages(&mut self, mut pages_ptr: Raw) { - assert_eq!(pages_ptr.refcount().load(Ordering::Relaxed), 0); - - let mut pfn = Into::<PFN>::into(pages_ptr); - let mut current_order = pages_ptr.order(); - - assert!( - pages_ptr.is_present(), - "Freeing a page that is not present: {:?}", - pages_ptr.into(), - ); - - assert!( - !pages_ptr.is_free(), - "Freeing a page that is free: {:?}", - pages_ptr.into(), - ); - - while current_order < (AREAS - 1) as u32 { - let buddy_pfn = pfn.buddy_pfn(current_order); - let buddy_pages_ptr = Raw::from(buddy_pfn); - - if !self.buddy_check(buddy_pages_ptr, current_order) { - break; - } - - pages_ptr.clear_buddy(); - buddy_pages_ptr.clear_buddy(); - self.free_areas[current_order as usize].del_pages(buddy_pages_ptr); - - pages_ptr = Raw::from(pfn.combined_pfn(buddy_pfn)); - pfn = pfn.combined_pfn(buddy_pfn); - - pages_ptr.set_buddy(); - current_order += 1; - } - - pages_ptr.set_order(current_order); - self.free_areas[current_order as usize].add_pages(pages_ptr); - } - - /// This function checks whether a page is free && is a buddy - /// we can coalesce a page and its buddy if - /// - the buddy is valid(present) && - /// - the buddy is right now in free_areas && - /// - a page and its buddy have the same order && - /// - a page and its buddy are in the same zone (on smp systems). - fn buddy_check(&self, pages_ptr: Raw, order: u32) -> bool { - if !pages_ptr.is_present() { - return false; - } - if !pages_ptr.is_free() { - return false; - } - if pages_ptr.order() != order { - return false; - } - - assert_eq!(pages_ptr.refcount().load(Ordering::Relaxed), 0); - true - } - - /// Only used on buddy initialization - pub fn create_pages(&mut self, start: PAddr, end: PAddr) { - let mut start_pfn = PFN::from(start.ceil()); - let end_pfn = PFN::from(end.floor()); - - while start_pfn < end_pfn { - let mut order = usize::from(start_pfn) - .trailing_zeros() - .min((AREAS - 1) as u32); - - while start_pfn + (1 << order) as usize > end_pfn { - order -= 1; - } - let page_ptr = Raw::from(start_pfn); - page_ptr.set_buddy(); - self.free_areas[order as usize].add_pages(page_ptr); - start_pfn = start_pfn + (1 << order) as usize; - } - } -} diff --git a/crates/eonix_hal/eonix_hal_traits/src/context.rs b/crates/eonix_hal/eonix_hal_traits/src/context.rs deleted file mode 100644 index 1d5e73ea..00000000 --- a/crates/eonix_hal/eonix_hal_traits/src/context.rs +++ /dev/null @@ -1,41 +0,0 @@ -#[doc(notable_trait)] -pub trait RawTaskContext: Sized { - /// Creates a new instance of the task context with interrupt enabled and the program - /// counter and stack pointer set to zero (a.k.a. some invalid state). - /// - /// Using the created context without setting the program counter and stack pointer - /// will result in undefined behavior. - fn new() -> Self; - - fn set_program_counter(&mut self, pc: usize); - fn set_stack_pointer(&mut self, sp: usize); - - fn is_interrupt_enabled(&self) -> bool; - fn set_interrupt_enabled(&mut self, is_enabled: bool); - - /// Sets the instruction pointer to the given function and prepares the context - /// to call it with the given argument. - fn call(&mut self, func: unsafe extern "C" fn(usize) -> !, arg: usize); - - /// Switch the execution context from `from` to `to`. - /// - /// # Safety - /// This function is unsafe because it performs a context switch, which can lead to - /// undefined behavior if the contexts are not properly set up. - unsafe extern "C" fn switch(from: &mut Self, to: &mut Self); - - /// Switches the execution context to `to` where we will not return. - /// - /// # Safety - /// This function is unsafe because it performs a context switch that does not return. - /// The caller must ensure that the `to` context is properly set up and that it will not - /// return to the caller. - unsafe extern "C" fn switch_to_noreturn(to: &mut Self) -> ! { - let mut from_ctx = Self::new(); - unsafe { - Self::switch(&mut from_ctx, to); - } - - unreachable!("We should never return from `switch_to_noreturn()`"); - } -} diff --git a/crates/eonix_hal/eonix_hal_traits/src/lib.rs b/crates/eonix_hal/eonix_hal_traits/src/lib.rs index a8cede83..5a5b4a23 100644 --- a/crates/eonix_hal/eonix_hal_traits/src/lib.rs +++ b/crates/eonix_hal/eonix_hal_traits/src/lib.rs @@ -1,7 +1,6 @@ #![no_std] #![feature(doc_notable_trait)] -pub mod context; pub mod fault; pub mod fpu; pub mod mm; diff --git a/crates/eonix_hal/eonix_hal_traits/src/trap.rs b/crates/eonix_hal/eonix_hal_traits/src/trap.rs index e51a9eb4..f7c6b149 100644 --- a/crates/eonix_hal/eonix_hal_traits/src/trap.rs +++ b/crates/eonix_hal/eonix_hal_traits/src/trap.rs @@ -1,7 +1,13 @@ -use crate::{context::RawTaskContext, fault::Fault}; use core::marker::PhantomData; + use eonix_mm::address::VAddr; +use crate::fault::Fault; + +pub trait Stack { + fn get_bottom(&self) -> *mut usize; +} + /// A raw trap context. /// /// This should be implemented by the architecture-specific trap context @@ -11,7 +17,26 @@ pub trait RawTrapContext: Copy { type FIrq: FnOnce(fn(irqno: usize)); type FTimer: FnOnce(fn()); - fn new() -> Self; + /// **Don't use this function unless you know what you're doing** + /// + /// Create a blank trap context. + /// + /// The context should be in a state that is ready to be used but whether + /// the interrupt is enabled or the context is in user mode is unspecified. + fn blank() -> Self; + + /// Create a new trap context. + /// + /// The context will be in a state that is ready to be used. Whether the + /// interrupt is enabled or the context is in user mode is specified by + /// the arguments. + fn new(int_enabled: bool, user: bool) -> Self { + let mut me = Self::blank(); + me.set_interrupt_enabled(int_enabled); + me.set_user_mode(user); + + me + } fn trap_type(&self) -> TrapType<Self::FIrq, Self::FTimer>; @@ -30,19 +55,18 @@ pub trait RawTrapContext: Copy { fn set_user_return_value(&mut self, retval: usize); fn set_user_call_frame<E>( - &mut self, - pc: usize, - sp: Option<usize>, - ra: Option<usize>, - args: &[usize], - write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, + &mut self, pc: usize, sp: Option<usize>, ra: Option<usize>, + args: &[usize], write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, ) -> Result<(), E>; + + fn set_kernel_call_frame( + &mut self, pc: usize, stack: &impl Stack, ra: Option<usize>, + args: &[usize], + ); } #[doc(notable_trait)] pub trait TrapReturn { - type TaskContext: RawTaskContext; - /// Return to the context before the trap occurred. /// /// # Safety @@ -51,6 +75,17 @@ pub trait TrapReturn { /// points to a valid stack frame and the program counter points to some /// valid instruction. unsafe fn trap_return(&mut self); + + /// Switch to the context before the trap occurred. + /// This function will NOT capture traps and will never return. + /// + /// # Safety + /// This function is unsafe because the caller MUST ensure that the + /// context before the trap is valid, that is, that the stack pointer + /// points to a valid stack frame and the program counter points to some + /// valid instruction. Besides, the caller MUST ensure that all variables + /// in the current context are released. + unsafe fn trap_return_noreturn(&mut self) -> !; } pub trait IrqState { diff --git a/crates/eonix_hal/src/arch/loongarch64/cpu.rs b/crates/eonix_hal/src/arch/loongarch64/cpu.rs index 612fbc3e..38e84fad 100644 --- a/crates/eonix_hal/src/arch/loongarch64/cpu.rs +++ b/crates/eonix_hal/src/arch/loongarch64/cpu.rs @@ -38,6 +38,7 @@ impl CPU { pub unsafe fn init(mut self: Pin<&mut Self>) { let me = self.as_mut().get_unchecked_mut(); setup_trap(); + super::interrupt::init(); } /// Boot all other hart. diff --git a/crates/eonix_hal/src/arch/loongarch64/interrupt/mod.rs b/crates/eonix_hal/src/arch/loongarch64/interrupt/mod.rs new file mode 100644 index 00000000..8409c4d2 --- /dev/null +++ b/crates/eonix_hal/src/arch/loongarch64/interrupt/mod.rs @@ -0,0 +1,163 @@ +//! External interrupt support for LoongArch64. +//! +//! On the QEMU `virt` machine the external interrupt path is: +//! +//! ```text +//! device (UART, ...) -> PCH-PIC line -> EIOINTC vector -> CPU HWI line +//! ``` +//! +//! The PCH-PIC is a normal MMIO device, while the EIOINTC (EXTIOI) lives in +//! the IOCSR address space and must be accessed with `iocsrrd`/`iocsrwr`. +//! +//! EIOINTC vectors are grouped: each group of 32 vectors is routed to one of +//! the four CPU hardware interrupt lines (HWI0..HWI3, i.e. CPU interrupt +//! numbers 2..5). We route every group to HWI1, which is the line the device +//! tree advertises for the interrupt controller (`interrupts = <3>`), and +//! every vector to CPU 0. + +use super::mm::ArchPhysAccess; +use eonix_mm::address::{PAddr, PhysAccess}; +use loongArch64::iocsr; + +/// Base address of the PCH-PIC (normal MMIO). +const PCH_PIC_BASE: usize = 0x1000_0000; + +const PCH_PIC_INT_MASK: usize = 0x020; +const PCH_PIC_HTMSI_VEC: usize = 0x200; + +/// Number of interrupt lines handled by the PCH-PIC. +const PCH_PIC_LINES: usize = 64; + +/// Number of EIOINTC vectors wired to PCH-PIC lines. +const PCH_PIC_VECTORS: usize = 32; + +/// EIOINTC registers, in the IOCSR address space. +const EIOINTC_IPMAP: usize = 0x14c0; +const EIOINTC_ENABLE: usize = 0x1600; +const EIOINTC_COREISR: usize = 0x1800; +const EIOINTC_COREMAP: usize = 0x1c00; + +const EIOINTC_VECTORS: usize = 256; +const EIOINTC_GROUP_SIZE: usize = 32; +const EIOINTC_GROUPS: usize = EIOINTC_VECTORS / EIOINTC_GROUP_SIZE; + +/// CPU interrupt line all EIOINTC vectors are routed to. +/// +/// This is the `ipnum` passed to the EIOINTC, whose outputs are connected to +/// CPU interrupt inputs `ipnum + 2` (HWI0 = 2). +const EIOINTC_IPNUM: usize = 1; + +fn pch_read32(offset: usize) -> u32 { + unsafe { + // SAFETY: The PCH-PIC registers are memory mapped at a well-known + // physical address on the QEMU virt platform. + ArchPhysAccess::as_ptr::<u32>(PAddr::from(PCH_PIC_BASE + offset)) + .as_ptr() + .read_volatile() + } +} + +fn pch_write32(offset: usize, value: u32) { + unsafe { + // SAFETY: See `pch_read32`. + ArchPhysAccess::as_ptr::<u32>(PAddr::from(PCH_PIC_BASE + offset)) + .as_ptr() + .write_volatile(value) + } +} + +fn pch_write8(offset: usize, value: u8) { + unsafe { + // SAFETY: See `pch_read32`. + ArchPhysAccess::as_ptr::<u8>(PAddr::from(PCH_PIC_BASE + offset)) + .as_ptr() + .write_volatile(value) + } +} + +fn eiointc_enable_vector(vector: usize) { + let offset = EIOINTC_ENABLE + (vector / EIOINTC_GROUP_SIZE) * size_of::<u32>(); + let bit = 1u32 << (vector % EIOINTC_GROUP_SIZE); + + iocsr::iocsr_write_w(offset, iocsr::iocsr_read_w(offset) | bit); +} + +fn eiointc_clear_vector(vector: usize) { + let offset = EIOINTC_COREISR + (vector / EIOINTC_GROUP_SIZE) * size_of::<u32>(); + let bit = 1u32 << (vector % EIOINTC_GROUP_SIZE); + + // Writing one acknowledges (clears) the pending bit. + iocsr::iocsr_write_w(offset, bit); +} + +/// Initialize the external interrupt controllers. +/// +/// Must be called once per CPU before interrupts are enabled. +pub fn init() { + // Route all 256 vectors to HWI1. The IPMAP register contains one byte + // per 32-vector group, holding a bitmask of the target `ipnum`. + let ipmap = 0x01u32 << EIOINTC_IPNUM; + let ipmap = ipmap * 0x0101_0101; + for group in 0..(EIOINTC_VECTORS / EIOINTC_GROUP_SIZE / 4) { + iocsr::iocsr_write_w(EIOINTC_IPMAP + group * size_of::<u32>(), ipmap); + } + + // Route all vectors to CPU 0. COREMAP stores four bytes (one vector + // each) per 32-bit register; a byte holds a bitmask of target CPUs. + for word in 0..(EIOINTC_VECTORS / 4) { + iocsr::iocsr_write_w(EIOINTC_COREMAP + word * size_of::<u32>(), 0x0101_0101); + } +} + +/// Enable an external interrupt line. +/// +/// `irq` is the PCH-PIC line number, which is also the EIOINTC vector number +/// for the lower 32 vectors. +pub fn enable_irq(irq: usize) { + assert!(irq < PCH_PIC_LINES, "PCH-PIC line out of range"); + + // Route this PCH-PIC line to the EIOINTC vector with the same number + // before unmasking it, so the first interrupt already lands on the + // correct vector. + pch_write8(PCH_PIC_HTMSI_VEC + irq, irq as u8); + + if irq < PCH_PIC_VECTORS { + eiointc_clear_vector(irq); + eiointc_enable_vector(irq); + } + + // Unmask the line in the PCH-PIC. + let offset = PCH_PIC_INT_MASK + (irq / 32) * size_of::<u32>(); + let bit = 1u32 << (irq % 32); + pch_write32(offset, pch_read32(offset) & !bit); +} + +/// Handle pending external interrupts by invoking `handler` for each of them. +pub fn dispatch_irq(handler: fn(usize)) { + for group in 0..EIOINTC_GROUPS { + let offset = EIOINTC_COREISR + group * size_of::<u32>(); + let mut pending = iocsr::iocsr_read_w(offset); + + if pending == 0 { + continue; + } + + // Acknowledge before dispatching, like Linux does. The peripheral + // keeps the line asserted until its handler has serviced it, and any + // new event after that re-raises the interrupt. + iocsr::iocsr_write_w(offset, pending); + + while pending != 0 { + let bit = pending.trailing_zeros() as usize; + pending &= pending - 1; + + let vector = group * EIOINTC_GROUP_SIZE + bit; + + // Vectors 0..32 are wired to the PCH-PIC. Higher vectors belong + // to the PCI MSI controller, which we do not support yet. + if vector < PCH_PIC_VECTORS { + handler(vector); + } + } + } +} diff --git a/crates/eonix_hal/src/arch/loongarch64/link.x b/crates/eonix_hal/src/arch/loongarch64/link.x index db95248e..e6bccdda 100644 --- a/crates/eonix_hal/src/arch/loongarch64/link.x +++ b/crates/eonix_hal/src/arch/loongarch64/link.x @@ -67,6 +67,30 @@ SECTIONS { RAW_SYSCALL_HANDLERS_SIZE = ABSOLUTE(__raw_syscall_handlers_end - __raw_syscall_handlers_start); } > REGION_RODATA AT> RAM + + .rodata.late_init : + { + . = ALIGN(16); + __raw_late_init_start = .; + + LATE_INIT_FUNCTIONS = .; + KEEP(*(.late_init)); + + __raw_late_init_end = .; + + __raw_late_init_async_start = .; + + LATE_INIT_ASYNC_FUNCTIONS = .; + KEEP(*(.late_init_async)); + + __raw_late_init_async_end = .; + + LATE_INIT_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_end - __raw_late_init_start) / 8); + + LATE_INIT_ASYNC_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_async_end - __raw_late_init_async_start) / 8); + } > REGION_RODATA AT> RAM } INSERT AFTER .rodata; diff --git a/crates/eonix_hal/src/arch/loongarch64/mod.rs b/crates/eonix_hal/src/arch/loongarch64/mod.rs index 46358d8d..da406a47 100644 --- a/crates/eonix_hal/src/arch/loongarch64/mod.rs +++ b/crates/eonix_hal/src/arch/loongarch64/mod.rs @@ -4,5 +4,6 @@ pub mod cpu; pub mod fdt; pub mod fence; pub mod fpu; +pub mod interrupt; pub mod mm; pub mod trap; diff --git a/crates/eonix_hal/src/arch/loongarch64/trap/trap_context.rs b/crates/eonix_hal/src/arch/loongarch64/trap/trap_context.rs index 56bf59b6..cc3da77f 100644 --- a/crates/eonix_hal/src/arch/loongarch64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/loongarch64/trap/trap_context.rs @@ -1,4 +1,4 @@ -use crate::{arch::trap::CSR_KERNEL_TP, processor::CPU}; +use crate::arch::trap::CSR_KERNEL_TP; use core::{arch::asm, mem::offset_of}; use eonix_hal_traits::{ fault::{Fault, PageFaultErrorCode}, @@ -151,18 +151,8 @@ impl RawTrapContext for TrapContext { }, }, Trap::Interrupt(interrupt) => match interrupt as usize { - 2..=7 => TrapType::Irq { - callback: |handler| { - todo!("handle IRQs"); - // let mut cpu = CPU::local(); - // match cpu.as_mut().interrupt.plic.claim_interrupt() { - // None => {} - // Some(irqno) => { - // cpu.interrupt.plic.complete_interrupt(irqno); - // handler(irqno); - // } - // } - }, + 2..=9 => TrapType::Irq { + callback: |handler| crate::arch::interrupt::dispatch_irq(handler), }, interrupt => TrapType::Fault(Fault::Unknown(interrupt)), }, diff --git a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs index 0f1dff63..b0c235aa 100644 --- a/crates/eonix_hal/src/arch/riscv64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/riscv64/bootstrap.rs @@ -1,39 +1,42 @@ -use super::{ - config::{self, mm::*}, - console::write_str, - cpu::{CPUID, CPU_COUNT}, - time::set_next_timer, -}; -use crate::{ - arch::{ - cpu::CPU, - fdt::{init_dtb_and_fdt, FdtExt, FDT}, - mm::{ArchPhysAccess, FreeRam, PageAttribute64, GLOBAL_PAGE_TABLE}, - }, - bootstrap::BootStrapData, - mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}, -}; -use core::{ - alloc::Allocator, - arch::asm, - cell::RefCell, - sync::atomic::{AtomicBool, AtomicUsize}, -}; -use core::{ - arch::{global_asm, naked_asm}, - hint::spin_loop, - sync::atomic::{AtomicPtr, Ordering}, -}; +use core::alloc::Allocator; +use core::arch::{asm, global_asm, naked_asm}; +use core::cell::RefCell; +use core::hint::spin_loop; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + use eonix_hal_traits::mm::Memory; -use eonix_mm::{ - address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}, - page_table::{PageAttribute, PagingMode, PTE as _}, - paging::{Page, PageAccess, PageAlloc, PAGE_SIZE, PFN}, +use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VAddr, VRange}; +use eonix_mm::page_table::{ + PageAttribute, PageTable, PagingMode, TableAttribute, PTE as _, +}; +use eonix_mm::paging::{ + Folio, FrameAlloc, PageAccess, PageBlock, PAGE_SIZE, PFN, }; use eonix_percpu::PercpuArea; use fdt::Fdt; -use riscv::{asm::sfence_vma_all, register::satp}; -use sbi::{hsm::hart_start, legacy::console_putchar, PhysicalAddress}; +use riscv::asm::sfence_vma_all; +use riscv::register::satp; +use sbi::hsm::hart_start; +use sbi::legacy::console_putchar; +use sbi::PhysicalAddress; + +use super::config::mm::*; +use super::config::{self}; +use super::console::write_str; +use super::cpu::{CPUID, CPU_COUNT}; +use super::time::set_next_timer; +use crate::arch::cpu::CPU; +use crate::arch::fdt::{init_dtb_and_fdt, FdtExt, FDT}; +use crate::arch::mm::{ + ArchPagingMode, ArchPhysAccess, PageAccessImpl, PageAttribute64, + RawPageTableSv48, GLOBAL_PAGE_TABLE, +}; +use crate::bootstrap::BootStrapData; +use crate::mm::{ + ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator, +}; +use crate::{extern_symbol_addr, extern_symbol_value}; #[unsafe(link_section = ".bootstrap.stack")] static BOOT_STACK: [u8; 4096 * 16] = [0; 4096 * 16]; @@ -46,29 +49,30 @@ static TEMP_AP_STACK: [u8; 256] = [0; 256]; static TEMP_AP_STACK_START: &'static [u8; 256] = &TEMP_AP_STACK; #[repr(C, align(4096))] -struct PageTable([u64; PTES_PER_PAGE]); +struct BootPageTable([u64; PTES_PER_PAGE]); /// map 0x8000 0000 to itself and 0xffff ffff 8000 0000 #[unsafe(link_section = ".bootstrap.page_table.1")] -static BOOT_PAGE_TABLE: PageTable = { +static BOOT_PAGE_TABLE: BootPageTable = { let mut arr: [u64; PTES_PER_PAGE] = [0; PTES_PER_PAGE]; arr[0] = 0 | 0x2f; arr[510] = 0 | 0x2f; arr[511] = (0x80202 << 10) | 0x21; - PageTable(arr) + BootPageTable(arr) }; #[unsafe(link_section = ".bootstrap.page_table.2")] #[used] -static PT1: PageTable = { +static PT1: BootPageTable = { let mut arr: [u64; PTES_PER_PAGE] = [0; PTES_PER_PAGE]; arr[510] = (0x80000 << 10) | 0x2f; - PageTable(arr) + BootPageTable(arr) }; -static BSP_PAGE_ALLOC: AtomicPtr<RefCell<BasicPageAlloc>> = AtomicPtr::new(core::ptr::null_mut()); +static BSP_PAGE_ALLOC: AtomicPtr<RefCell<BasicPageAlloc>> = + AtomicPtr::new(core::ptr::null_mut()); static AP_COUNT: AtomicUsize = AtomicUsize::new(0); static AP_STACK: AtomicUsize = AtomicUsize::new(0); @@ -78,7 +82,7 @@ static AP_SEM: AtomicBool = AtomicBool::new(false); #[unsafe(naked)] #[unsafe(no_mangle)] #[unsafe(link_section = ".bootstrap.entry")] -unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) -> ! { +unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) { naked_asm!( " ld sp, 2f @@ -109,17 +113,16 @@ unsafe extern "C" fn _start(hart_id: usize, dtb_addr: usize) -> ! { } pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { - let fdt = Fdt::from_ptr(ArchPhysAccess::as_ptr(dtb_addr).as_ptr()) - .expect("Failed to parse DTB from static memory."); + let fdt = unsafe { FdtExt::new(dtb_addr) }; let real_allocator = RefCell::new(BasicPageAlloc::new()); let alloc = BasicPageAllocRef::new(&real_allocator); - for range in fdt.present_ram().free_ram() { + for range in fdt.free_ram() { real_allocator.borrow_mut().add_range(range); } - setup_kernel_page_table(&alloc); + setup_kernel_page_table(alloc.clone()); unsafe { init_dtb_and_fdt(dtb_addr); } @@ -134,11 +137,14 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { } let start = unsafe { - ((&BOOT_STACK_START) as *const &'static [u8; 4096 * 16]).read_volatile() as *const _ - as usize + ((&BOOT_STACK_START) as *const &'static [u8; 4096 * 16]).read_volatile() + as *const _ as usize }; let bootstrap_data = BootStrapData { - early_stack: PRange::new(PAddr::from(start), PAddr::from(start + 4096 * 16)), + early_stack: PRange::new( + PAddr::from(start), + PAddr::from(start + 4096 * 16), + ), allocator: Some(real_allocator), }; @@ -150,43 +156,40 @@ pub unsafe extern "C" fn riscv64_start(hart_id: usize, dtb_addr: PAddr) -> ! { } } -unsafe extern "C" { - fn BSS_LENGTH(); -} - /// TODO: /// 对kernel image添加更细的控制,或者不加也行 -fn setup_kernel_page_table(alloc: impl PageAlloc) { - let global_page_table = &GLOBAL_PAGE_TABLE; +fn setup_kernel_page_table(alloc: BasicPageAllocRef) { + let global_page_table = PageTable::<ArchPagingMode, _, _>::new( + GLOBAL_PAGE_TABLE.clone(), + alloc.clone(), + PageAccessImpl, + ); let attr = PageAttribute::WRITE | PageAttribute::READ - | PageAttribute::EXECUTE | PageAttribute::GLOBAL | PageAttribute::PRESENT; const KERNEL_BSS_START: VAddr = VAddr::from(0xffffffff40000000); + let bss_length = extern_symbol_addr!(BSS_LENGTH); + // Map kernel BSS - for pte in global_page_table.iter_kernel_in( - VRange::from(KERNEL_BSS_START).grow(BSS_LENGTH as usize), - ArchPagingMode::LEVELS, - &alloc, - ) { - let page = Page::alloc_in(&alloc); - - let attr = { - let mut attr = attr.clone(); - attr.remove(PageAttribute::EXECUTE); - attr - }; + let bss_range = VRange::from(KERNEL_BSS_START).grow(bss_length); + for pte in global_page_table.iter_kernel(bss_range) { + let page = alloc.alloc().unwrap(); + pte.set(page.into_raw(), attr.into()); } sfence_vma_all(); unsafe { - core::ptr::write_bytes(KERNEL_BSS_START.addr() as *mut (), 0, BSS_LENGTH as usize); + core::ptr::write_bytes( + KERNEL_BSS_START.addr() as *mut u8, + 0, + bss_length, + ); } unsafe { @@ -197,17 +200,22 @@ fn setup_kernel_page_table(alloc: impl PageAlloc) { ); } sfence_vma_all(); + + core::mem::forget(global_page_table); } /// set up tp register to percpu -fn setup_cpu(alloc: impl PageAlloc, hart_id: usize) { +fn setup_cpu(alloc: impl FrameAlloc, hart_id: usize) { CPU_COUNT.fetch_add(1, Ordering::Relaxed); let mut percpu_area = PercpuArea::new(|layout| { let page_count = layout.size().div_ceil(PAGE_SIZE); - let page = Page::alloc_at_least_in(page_count, alloc); + let page = alloc.alloc_at_least(page_count).unwrap(); - let ptr = ArchPhysAccess::get_ptr_for_page(&page).cast(); + let ptr = unsafe { + // TODO: safety + ArchPhysAccess::as_ptr(page.start()) + }; page.into_raw(); ptr @@ -235,15 +243,6 @@ fn setup_cpu(alloc: impl PageAlloc, hart_id: usize) { percpu_area.register(cpu.cpuid()); } -fn get_ap_start_addr() -> usize { - unsafe extern "C" { - fn _ap_start(); - } - static AP_START_VALUE: &'static unsafe extern "C" fn() = - &(_ap_start as unsafe extern "C" fn()); - unsafe { (AP_START_VALUE as *const _ as *const usize).read_volatile() } -} - fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { let local_hart_id = CPU::local().cpuid(); let mut ap_count = 0; @@ -251,13 +250,14 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { for hart_id in FDT.harts().filter(|&id| id != local_hart_id) { let stack_range = { let page_alloc = BasicPageAllocRef::new(&page_alloc); - let ap_stack = Page::alloc_order_in(4, page_alloc); + let ap_stack = page_alloc.alloc_order(4).unwrap(); let stack_range = ap_stack.range(); ap_stack.into_raw(); stack_range }; - let old = BSP_PAGE_ALLOC.swap((&raw const *page_alloc) as *mut _, Ordering::Release); + let old = BSP_PAGE_ALLOC + .swap((&raw const *page_alloc) as *mut _, Ordering::Release); assert!(old.is_null()); while AP_STACK @@ -273,7 +273,11 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { } unsafe { - hart_start(hart_id, PhysicalAddress::new(get_ap_start_addr()), 0); + hart_start( + hart_id, + PhysicalAddress::new(extern_symbol_value!(_ap_start)), + 0, + ); } while AP_COUNT.load(Ordering::Acquire) == ap_count { @@ -289,7 +293,7 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { #[unsafe(naked)] #[unsafe(no_mangle)] #[unsafe(link_section = ".bootstrap.apentry")] -unsafe extern "C" fn _ap_start(hart_id: usize) -> ! { +unsafe extern "C" fn _ap_start(hart_id: usize) { naked_asm!( " la sp, 1f // set temp stack @@ -326,7 +330,12 @@ unsafe extern "C" fn _ap_start(hart_id: usize) -> ! { fn get_ap_stack() -> usize { while AP_SEM - .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .compare_exchange_weak( + false, + true, + Ordering::Acquire, + Ordering::Relaxed, + ) .is_err() { core::hint::spin_loop(); @@ -346,12 +355,14 @@ fn get_ap_stack() -> usize { } fn ap_entry(hart_id: usize, stack_bottom: PAddr) -> ! { - let stack_range = PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); + let stack_range = + PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); { // SAFETY: Acquire all the work done by the BSP and other APs. let alloc = loop { - let alloc = BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); + let alloc = + BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); if !alloc.is_null() { break alloc; diff --git a/crates/eonix_hal/src/arch/riscv64/context.rs b/crates/eonix_hal/src/arch/riscv64/context.rs deleted file mode 100644 index e7a53ea3..00000000 --- a/crates/eonix_hal/src/arch/riscv64/context.rs +++ /dev/null @@ -1,111 +0,0 @@ -use core::arch::naked_asm; -use eonix_hal_traits::context::RawTaskContext; -use riscv::register::sstatus::Sstatus; - -#[repr(C)] -#[derive(Debug)] -pub struct TaskContext { - // s0-11 - s: [u64; 12], - sp: u64, - ra: u64, - sstatus: Sstatus, -} - -impl RawTaskContext for TaskContext { - fn new() -> Self { - Self::new() - } - - fn set_program_counter(&mut self, pc: usize) { - self.ra = pc as u64; - } - - fn set_stack_pointer(&mut self, sp: usize) { - self.sp = sp as u64; - } - - fn is_interrupt_enabled(&self) -> bool { - self.sstatus.sie() - } - - fn set_interrupt_enabled(&mut self, is_enabled: bool) { - self.sstatus.set_sie(is_enabled); - } - - fn call(&mut self, func: unsafe extern "C" fn(usize) -> !, arg: usize) { - self.s[0] = func as u64; - self.s[1] = arg as u64; - - self.set_program_counter(Self::do_call as usize); - } - - #[unsafe(naked)] - unsafe extern "C" fn switch(from: &mut Self, to: &mut Self) { - // Input arguments `from` and `to` will be in `a0` (x10) and `a1` (x11). - naked_asm!( - // Save current task's callee-saved registers to `from` context - "sd s0, 0(a0)", - "sd s1, 8(a0)", - "sd s2, 16(a0)", - "sd s3, 24(a0)", - "sd s4, 32(a0)", - "sd s5, 40(a0)", - "sd s6, 48(a0)", - "sd s7, 56(a0)", - "sd s8, 64(a0)", - "sd s9, 72(a0)", - "sd s10, 80(a0)", - "sd s11, 88(a0)", - "sd sp, 96(a0)", - "sd ra, 104(a0)", - "csrr t0, sstatus", - "sd t0, 112(a0)", - "", - "ld s0, 0(a1)", - "ld s1, 8(a1)", - "ld s2, 16(a1)", - "ld s3, 24(a1)", - "ld s4, 32(a1)", - "ld s5, 40(a1)", - "ld s6, 48(a1)", - "ld s7, 56(a1)", - "ld s8, 64(a1)", - "ld s9, 72(a1)", - "ld s10, 80(a1)", - "ld s11, 88(a1)", - "ld sp, 96(a1)", - "ld ra, 104(a1)", - "ld t0, 112(a1)", - "csrw sstatus, t0", - "ret", - ); - } -} - -impl TaskContext { - pub const fn new() -> Self { - Self { - s: [0; 12], - sp: 0, - ra: 0, - sstatus: Sstatus::from_bits((1 << 13) | (1 << 18)), // FS = Initial, SUM = 1. - } - } - - #[unsafe(naked)] - /// Maximum of 5 arguments supported. - unsafe extern "C" fn do_call() -> ! { - naked_asm!( - "mv t0, s0", // Function pointer in s0. - "mv a0, s1", // Args - "mv a1, s2", - "mv a2, s3", - "mv a3, s4", - "mv a4, s5", - "mv fp, zero", // Set frame pointer to 0. - "mv ra, zero", - "jr t0", - ); - } -} diff --git a/crates/eonix_hal/src/arch/riscv64/cpu.rs b/crates/eonix_hal/src/arch/riscv64/cpu.rs index 9c843eaf..3c58580e 100644 --- a/crates/eonix_hal/src/arch/riscv64/cpu.rs +++ b/crates/eonix_hal/src/arch/riscv64/cpu.rs @@ -27,22 +27,11 @@ static DEFAULT_TRAP_CONTEXT: MaybeUninit<TrapContext> = MaybeUninit::uninit(); #[eonix_percpu::define_percpu] static LOCAL_CPU: LazyLock<CPU> = LazyLock::new(|| CPU::new(CPUID.get())); -#[derive(Debug, Clone)] -pub enum UserTLS { - Base(u64), -} - /// RISC-V Hart pub struct CPU { pub(crate) interrupt: InterruptControl, } -impl UserTLS { - pub fn new(base: u64) -> Self { - Self::Base(base) - } -} - impl CPU { fn new(cpuid: usize) -> Self { Self { @@ -66,12 +55,6 @@ impl CPU { sscratch::write(DEFAULT_TRAP_CONTEXT.as_ptr() as usize); } - pub unsafe fn load_interrupt_stack(self: Pin<&mut Self>, sp: u64) {} - - pub fn set_tls32(self: Pin<&mut Self>, _user_tls: &UserTLS) { - // nothing - } - pub fn local() -> PreemptGuard<Pin<&'static mut Self>> { unsafe { // SAFETY: We pass the reference into a `PreemptGuard`, which ensures diff --git a/crates/eonix_hal/src/arch/riscv64/fdt.rs b/crates/eonix_hal/src/arch/riscv64/fdt.rs index 5efcc98d..908256c7 100644 --- a/crates/eonix_hal/src/arch/riscv64/fdt.rs +++ b/crates/eonix_hal/src/arch/riscv64/fdt.rs @@ -1,62 +1,95 @@ -use super::mm::{ArchPhysAccess, PresentRam}; -use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; -use core::sync::atomic::{AtomicPtr, Ordering}; -use eonix_mm::address::{PAddr, PRange, PhysAccess}; +use core::ops::Deref; +use core::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; + +use eonix_mm::address::{Addr, AddrOps, PAddr, PRange, PhysAccess}; use eonix_sync_base::LazyLock; use fdt::Fdt; -static DTB_VIRT_PTR: AtomicPtr<u8> = AtomicPtr::new(core::ptr::null_mut()); -pub static FDT: LazyLock<Fdt<'static>> = LazyLock::new(|| unsafe { - Fdt::from_ptr(DTB_VIRT_PTR.load(Ordering::Acquire)) - .expect("Failed to parse DTB from static memory.") +use super::mm::ArchPhysAccess; +use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; +use crate::extern_symbol_addr; + +static DTB_PADDR: AtomicUsize = AtomicUsize::new(0); +pub static FDT: LazyLock<FdtExt> = LazyLock::new(|| unsafe { + FdtExt::new(PAddr::from_val(DTB_PADDR.load(Ordering::Relaxed))) }); -pub trait FdtExt { - fn harts(&self) -> impl Iterator<Item = usize>; +pub struct FdtExt { + fdt: Fdt<'static>, + range: PRange, +} - fn hart_count(&self) -> usize { - self.harts().count() - } +impl FdtExt { + /// # Safety + /// The caller MUST ensure that [`addr`] points to valid FDT. + pub unsafe fn new(addr: PAddr) -> Self { + let fdt = unsafe { + Fdt::from_ptr(ArchPhysAccess::as_ptr(addr).as_ptr()) + .expect("Failed to parse DTB from static memory.") + }; - fn present_ram(&self) -> impl Iterator<Item = PRange>; -} + Self { + range: PRange::from(addr).grow(fdt.total_size()), + fdt, + } + } -impl FdtExt for Fdt<'_> { - fn harts(&self) -> impl Iterator<Item = usize> { + pub fn harts(&self) -> impl Iterator<Item = usize> { self.cpus().map(|cpu| cpu.ids().all()).flatten() } - fn present_ram(&self) -> impl Iterator<Item = PRange> + PresentRam { - struct Present<I>(I); - impl<I> PresentRam for Present<I> where I: Iterator<Item = PRange> {} - impl<I> Iterator for Present<I> - where - I: Iterator<Item = PRange>, - { - type Item = PRange; - - fn next(&mut self) -> Option<Self::Item> { - self.0.next() - } - } + pub fn hart_count(&self) -> usize { + self.harts().count() + } + pub fn present_ram(&self) -> impl Iterator<Item = PRange> { let mut index = 0; - Present(core::iter::from_fn(move || { - self.memory() + + core::iter::from_fn(move || { + let item = self + .memory() .regions() .filter_map(|region| { - region.size.map(|len| { - PRange::from(PAddr::from(region.starting_address as usize)).grow(len) - }) + let start = PAddr::from(region.starting_address as usize); + Some(start).zip(region.size) }) - .skip(index) - .next() - .inspect(|_| index += 1) - })) + .map(|(start, len)| PRange::from(start).grow(len)) + .nth(index); + + index += 1; + item + }) + } + + pub fn free_ram(&self) -> impl Iterator<Item = PRange> { + let kernel_end = extern_symbol_addr!(__kernel_end) - KIMAGE_OFFSET; + let kernel_end = PAddr::from(kernel_end).ceil(); + + // TODO: move this to some platform-specific crate + self.present_ram().map(move |mut range| { + // Strip out parts before __kernel_end + if range.overlap_with(&PRange::from(kernel_end)) { + (_, range) = range.split_at(kernel_end); + } + + // Strip out part after the FDT + if range.overlap_with(&self.range) { + (range, _) = range.split_at(self.range.start()); + } + + range + }) + } +} + +impl Deref for FdtExt { + type Target = Fdt<'static>; + + fn deref(&self) -> &Self::Target { + &self.fdt } } pub unsafe fn init_dtb_and_fdt(dtb_paddr: PAddr) { - let dtb_virt_ptr = ArchPhysAccess::as_ptr(dtb_paddr); - DTB_VIRT_PTR.store(dtb_virt_ptr.as_ptr(), Ordering::Release); + DTB_PADDR.store(dtb_paddr.addr(), Ordering::Relaxed); } diff --git a/crates/eonix_hal/src/arch/riscv64/link.x b/crates/eonix_hal/src/arch/riscv64/link.x index e348e1be..f7cb8f7f 100644 --- a/crates/eonix_hal/src/arch/riscv64/link.x +++ b/crates/eonix_hal/src/arch/riscv64/link.x @@ -62,6 +62,30 @@ SECTIONS { RAW_SYSCALL_HANDLERS_SIZE = ABSOLUTE(__raw_syscall_handlers_end - __raw_syscall_handlers_start); } > REGION_RODATA AT> RAM + + .rodata.late_init : + { + . = ALIGN(16); + __raw_late_init_start = .; + + LATE_INIT_FUNCTIONS = .; + KEEP(*(.late_init)); + + __raw_late_init_end = .; + + __raw_late_init_async_start = .; + + LATE_INIT_ASYNC_FUNCTIONS = .; + KEEP(*(.late_init_async)); + + __raw_late_init_async_end = .; + + LATE_INIT_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_end - __raw_late_init_start) / 8); + + LATE_INIT_ASYNC_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_async_end - __raw_late_init_async_start) / 8); + } > REGION_RODATA AT> RAM } INSERT AFTER .rodata; @@ -81,10 +105,12 @@ INSERT AFTER .rodata; SECTIONS { .vdso ALIGN(0x1000) : ALIGN(0x1000) { + VDSO_START = ABSOLUTE(.); + KEEP(*(.vdso .vdso.*)); . = ALIGN(0x1000); - } > VDSO AT> RAM + } > REGION_DATA AT> RAM VDSO_PADDR = LOADADDR(.vdso); } diff --git a/crates/eonix_hal/src/arch/riscv64/memory.x b/crates/eonix_hal/src/arch/riscv64/memory.x index 0dc7c4ff..f2029c9a 100644 --- a/crates/eonix_hal/src/arch/riscv64/memory.x +++ b/crates/eonix_hal/src/arch/riscv64/memory.x @@ -3,7 +3,6 @@ ENTRY(_start) MEMORY { RAM : org = 0x0000000080200000, len = 8M - VDSO : org = 0x00007f0000000000, len = 4K KBSS : org = 0xffffffff40000000, len = 2M KIMAGE : org = 0xffffffff80200000, len = 8M } diff --git a/crates/eonix_hal/src/arch/riscv64/mm.rs b/crates/eonix_hal/src/arch/riscv64/mm.rs index 46dd9437..7891f094 100644 --- a/crates/eonix_hal/src/arch/riscv64/mm.rs +++ b/crates/eonix_hal/src/arch/riscv64/mm.rs @@ -1,31 +1,26 @@ -use super::{ - config::mm::{PHYS_MAP_VIRT, ROOT_PAGE_TABLE_PFN}, - fdt::{FdtExt, FDT}, -}; -use crate::{arch::riscv64::config::mm::KIMAGE_OFFSET, traits::mm::Memory}; -use core::{marker::PhantomData, ptr::NonNull}; -use eonix_mm::{ - address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}, - page_table::{ - PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, - TableAttribute, PTE, - }, - paging::{NoAlloc, Page, PageBlock, PFN}, +use core::marker::PhantomData; +use core::ptr::NonNull; + +use eonix_hal_traits::mm::Memory; +use eonix_mm::address::{Addr as _, AddrOps, PAddr, PRange, PhysAccess, VAddr}; +use eonix_mm::page_table::{ + PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, + RawPageTable, TableAttribute, PTE, }; +use eonix_mm::paging::{BasicFolio, Folio, PageAccess, PageBlock, PFN}; use eonix_sync_base::LazyLock; use fdt::Fdt; -use riscv::{ - asm::{sfence_vma, sfence_vma_all}, - register::satp, -}; +use riscv::asm::{sfence_vma, sfence_vma_all}; +use riscv::register::satp; + +use super::config::mm::{PHYS_MAP_VIRT, ROOT_PAGE_TABLE_PFN}; +use super::fdt::{FdtExt, FDT}; +use crate::arch::riscv64::config::mm::KIMAGE_OFFSET; +use crate::extern_symbol_addr; +use crate::mm::BasicPageAlloc; -pub const PAGE_TABLE_BASE: PFN = PFN::from_val(ROOT_PAGE_TABLE_PFN); -pub static GLOBAL_PAGE_TABLE: LazyLock<PageTable<ArchPagingMode, NoAlloc, ArchPhysAccess>> = - LazyLock::new(|| unsafe { - Page::with_raw(PAGE_TABLE_BASE, |root_table_page| { - PageTable::with_root_table(root_table_page.clone()) - }) - }); +const PAGE_TABLE_BASE: PFN = PFN::from_val(ROOT_PAGE_TABLE_PFN); +pub const GLOBAL_PAGE_TABLE: BasicFolio = BasicFolio::new(PAGE_TABLE_BASE, 0); pub const PA_V: u64 = 0b1 << 0; pub const PA_R: u64 = 0b1 << 1; @@ -61,6 +56,9 @@ pub struct ArchPhysAccess; pub struct ArchMemory; +#[derive(Clone)] +pub struct PageAccessImpl; + impl PTE for PTE64 { type Attr = PageAttribute64; @@ -118,7 +116,9 @@ impl RawAttribute for PageAttribute64 { table_attr |= TableAttribute::PRESENT; } - if table_attr.contains(TableAttribute::PRESENT) && self.0 & (PA_R | PA_W | PA_X) != 0 { + if table_attr.contains(TableAttribute::PRESENT) + && self.0 & (PA_R | PA_W | PA_X) != 0 + { return None; } @@ -142,7 +142,9 @@ impl RawAttribute for PageAttribute64 { page_attr |= PageAttribute::PRESENT; } - if page_attr.contains(PageAttribute::PRESENT) && (self.0 & (PA_R | PA_W | PA_X) == 0) { + if page_attr.contains(PageAttribute::PRESENT) + && (self.0 & (PA_R | PA_W | PA_X) == 0) + { return None; } @@ -261,70 +263,24 @@ impl PhysAccess for ArchPhysAccess { } } +impl PageAccess for PageAccessImpl { + unsafe fn get_ptr_for_pfn(&self, pfn: PFN) -> NonNull<PageBlock> { + unsafe { ArchPhysAccess::as_ptr(PAddr::from(pfn)) } + } +} + impl Memory for ArchMemory { fn present_ram() -> impl Iterator<Item = PRange> { FDT.present_ram() } fn free_ram() -> impl Iterator<Item = PRange> { - unsafe extern "C" { - fn __kernel_start(); - fn __kernel_end(); - } - - let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); - let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)).chain( - Self::present_ram() - .filter(move |range| range.end() > paddr_after_kimage_aligned) - .map(move |range| { - if range.start() < paddr_after_kimage_aligned { - let (_, right) = range.split_at(paddr_after_kimage_aligned); - right - } else { - range - } - }), - ) + FDT.free_ram() } } pub type DefaultPagingMode = PagingModeSv48; -pub trait PresentRam: Iterator<Item = PRange> {} - -pub trait FreeRam: PresentRam { - fn free_ram(self) -> impl Iterator<Item = PRange>; -} - -impl<T> FreeRam for T -where - T: PresentRam, -{ - fn free_ram(self) -> impl Iterator<Item = PRange> { - unsafe extern "C" { - fn __kernel_start(); - fn __kernel_end(); - } - - let kernel_end = PAddr::from(__kernel_end as usize - KIMAGE_OFFSET); - let paddr_after_kimage_aligned = kernel_end.ceil_to(0x200000); - - core::iter::once(PRange::new(kernel_end, paddr_after_kimage_aligned)).chain( - self.filter(move |range| range.end() > paddr_after_kimage_aligned) - .map(move |range| { - if range.start() < paddr_after_kimage_aligned { - let (_, right) = range.split_at(paddr_after_kimage_aligned); - right - } else { - range - } - }), - ) - } -} - #[inline(always)] pub fn flush_tlb(vaddr: usize) { sfence_vma(0, vaddr); diff --git a/crates/eonix_hal/src/arch/riscv64/mod.rs b/crates/eonix_hal/src/arch/riscv64/mod.rs index a933c1a7..49574cce 100644 --- a/crates/eonix_hal/src/arch/riscv64/mod.rs +++ b/crates/eonix_hal/src/arch/riscv64/mod.rs @@ -1,12 +1,11 @@ pub mod bootstrap; mod config; pub mod console; -pub mod context; pub mod cpu; pub mod fdt; pub mod fence; pub mod fpu; pub mod interrupt; pub mod mm; -pub mod trap; pub mod time; +pub mod trap; diff --git a/crates/eonix_hal/src/arch/riscv64/trap/captured.rs b/crates/eonix_hal/src/arch/riscv64/trap/captured.rs index d4c00e9f..00b3a35a 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/captured.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/captured.rs @@ -1,61 +1,110 @@ -use crate::{arch::trap::Registers, context::TaskContext, trap::TrapContext}; -use core::{arch::naked_asm, mem::MaybeUninit}; -use eonix_hal_traits::context::RawTaskContext; +use core::arch::naked_asm; +use core::mem::MaybeUninit; -static mut DIRTY_TASK_CONTEXT: MaybeUninit<TaskContext> = MaybeUninit::uninit(); +use crate::arch::trap::Registers; +use crate::trap::TrapContext; // If captured trap context is present, we use it directly. -// We need to restore the kernel tp from that TrapContext but sp is -// fine since we will use TaskContext::switch. +// We need to restore the callee saved registers from that TrapContext. #[unsafe(naked)] pub(super) unsafe extern "C" fn _captured_trap_entry() -> ! { naked_asm!( - "csrrw t0, sscratch, t0", - "sd tp, {tp}(t0)", - "ld tp, {ra}(t0)", // Load kernel tp from trap_ctx.ra - "sd ra, {ra}(t0)", - "ld ra, {sp}(t0)", // Load capturer task context from trap_ctx.sp - "sd sp, {sp}(t0)", - "sd gp, {gp}(t0)", - "sd a0, {a0}(t0)", - "sd a1, {a1}(t0)", - "sd a2, {a2}(t0)", - "sd a3, {a3}(t0)", - "sd a4, {a4}(t0)", - "sd t1, {t1}(t0)", - "sd a5, {a5}(t0)", - "sd a6, {a6}(t0)", - "sd a7, {a7}(t0)", - "sd t3, {t3}(t0)", - "sd t4, {t4}(t0)", - "sd t5, {t5}(t0)", - "sd t2, {t2}(t0)", - "sd t6, {t6}(t0)", - "sd s0, {s0}(t0)", - "sd s1, {s1}(t0)", - "sd s2, {s2}(t0)", - "sd s3, {s3}(t0)", - "sd s4, {s4}(t0)", - "sd s5, {s5}(t0)", - "sd s6, {s6}(t0)", - "sd s7, {s7}(t0)", - "sd s8, {s8}(t0)", - "sd s9, {s9}(t0)", - "sd s10, {s10}(t0)", - "sd s11, {s11}(t0)", - "csrr t2, sstatus", - "csrr t3, sepc", - "csrr t4, scause", - "csrr t5, stval", - "csrrw t6, sscratch, t0", - "sd t6, {t0}(t0)", - "sd t2, {sstatus}(t0)", - "sd t3, {sepc}(t0)", - "sd t4, {scause}(t0)", - "sd t5, {stval}(t0)", - "la a0, {dirty_task_context}", - "mv a1, ra", - "j {task_context_switch}", + "csrrw gp, sscratch, gp", + + "sd a0, {a0}(gp)", + "mv a0, s0", + "ld s0, {s0}(gp)", + "sd a0, {s0}(gp)", + + "sd a1, {a1}(gp)", + "mv a1, s1", + "ld s1, {s1}(gp)", + "sd a1, {s1}(gp)", + + "sd a2, {a2}(gp)", + "mv a2, s2", + "ld s2, {s2}(gp)", + "sd a2, {s2}(gp)", + + "sd a3, {a3}(gp)", + "mv a3, s3", + "ld s3, {s3}(gp)", + "sd a3, {s3}(gp)", + + "sd a4, {a4}(gp)", + "mv a4, s4", + "ld s4, {s4}(gp)", + "sd a4, {s4}(gp)", + + "sd a5, {a5}(gp)", + "mv a5, s5", + "ld s5, {s5}(gp)", + "sd a5, {s5}(gp)", + + "sd a6, {a6}(gp)", + "mv a6, s6", + "ld s6, {s6}(gp)", + "sd a6, {s6}(gp)", + + "sd a7, {a7}(gp)", + "mv a7, s7", + "ld s7, {s7}(gp)", + "sd a7, {s7}(gp)", + + "sd t0, {t0}(gp)", + "mv t0, s8", + "ld s8, {s8}(gp)", + "sd t0, {s8}(gp)", + + "sd t1, {t1}(gp)", + "mv t1, s9", + "ld s9, {s9}(gp)", + "sd t1, {s9}(gp)", + + "sd t2, {t2}(gp)", + "mv t2, s10", + "ld s10, {s10}(gp)", + "sd t2, {s10}(gp)", + + "sd t3, {t3}(gp)", + "mv t3, s11", + "ld s11, {s11}(gp)", + "sd t3, {s11}(gp)", + + "sd t4, {t4}(gp)", + "mv t4, ra", + "ld ra, {ra}(gp)", + "sd t4, {ra}(gp)", + + "sd t5, {t5}(gp)", + "mv t5, tp", + "ld tp, {tp}(gp)", + "sd t5, {tp}(gp)", + + "sd t6, {t6}(gp)", + "mv t6, sp", + "ld sp, {sp}(gp)", + "sd t6, {sp}(gp)", + + "ld t5, {sstatus}(gp)", + + "csrrw t0, sscratch, gp", + "sd t0, {gp}(gp)", + + "csrr t1, sstatus", + "sd t1, {sstatus}(gp)", + + "csrr t2, sepc", + "sd t2, {sepc}(gp)", + + "csrr t3, scause", + "sd t3, {scause}(gp)", + + "csrr t4, stval", + "sd t4, {stval}(gp)", + + "csrw sstatus, t5", + "ret", ra = const Registers::OFFSET_RA, sp = const Registers::OFFSET_SP, gp = const Registers::OFFSET_GP, @@ -91,54 +140,99 @@ pub(super) unsafe extern "C" fn _captured_trap_entry() -> ! { sepc = const TrapContext::OFFSET_SEPC, scause = const TrapContext::OFFSET_SCAUSE, stval = const TrapContext::OFFSET_STVAL, - dirty_task_context = sym DIRTY_TASK_CONTEXT, - task_context_switch = sym TaskContext::switch, ); } #[unsafe(naked)] -pub(super) unsafe extern "C" fn _captured_trap_return(ctx: &mut TrapContext) -> ! { +pub(super) unsafe extern "C" fn _captured_trap_return(ctx: &mut TrapContext) { naked_asm!( - "csrr t0, sscratch", - "ld t1, {sstatus}(t0)", - "ld t2, {sepc}(t0)", - "csrw sstatus, t1", - "csrw sepc, t2", - "mv t4, tp", - "mv t5, sp", - "ld tp, {tp}(t0)", - "ld ra, {ra}(t0)", - "ld sp, {sp}(t0)", - "sd t4, {ra}(t0)", // Store kernel tp to trap_ctx.ra - "sd t5, {sp}(t0)", // Store capturer task context to trap_ctx.sp - "ld gp, {gp}(t0)", - "ld a0, {a0}(t0)", - "ld a1, {a1}(t0)", - "ld a2, {a2}(t0)", - "ld a3, {a3}(t0)", - "ld a4, {a4}(t0)", - "ld t1, {t1}(t0)", - "ld a5, {a5}(t0)", - "ld a6, {a6}(t0)", - "ld a7, {a7}(t0)", - "ld t3, {t3}(t0)", - "ld t4, {t4}(t0)", - "ld t5, {t5}(t0)", - "ld t2, {t2}(t0)", - "ld t6, {t6}(t0)", - "ld s0, {s0}(t0)", - "ld s1, {s1}(t0)", - "ld s2, {s2}(t0)", - "ld s3, {s3}(t0)", - "ld s4, {s4}(t0)", - "ld s5, {s5}(t0)", - "ld s6, {s6}(t0)", - "ld s7, {s7}(t0)", - "ld s8, {s8}(t0)", - "ld s9, {s9}(t0)", - "ld s10, {s10}(t0)", - "ld s11, {s11}(t0)", - "ld t0, {t0}(t0)", + "mv t6, a0", + + "mv a0, s0", + "ld s0, {s0}(t6)", + "sd a0, {s0}(t6)", + + "mv a1, s1", + "ld s1, {s1}(t6)", + "sd a1, {s1}(t6)", + + "mv a2, s2", + "ld s2, {s2}(t6)", + "sd a2, {s2}(t6)", + + "mv a3, s3", + "ld s3, {s3}(t6)", + "sd a3, {s3}(t6)", + + "mv a4, s4", + "ld s4, {s4}(t6)", + "sd a4, {s4}(t6)", + + "mv a5, s5", + "ld s5, {s5}(t6)", + "sd a5, {s5}(t6)", + + "mv a6, s6", + "ld s6, {s6}(t6)", + "sd a6, {s6}(t6)", + + "mv a7, s7", + "ld s7, {s7}(t6)", + "sd a7, {s7}(t6)", + + "mv t0, s8", + "ld s8, {s8}(t6)", + "sd t0, {s8}(t6)", + + "mv t1, s9", + "ld s9, {s9}(t6)", + "sd t1, {s9}(t6)", + + "mv t2, s10", + "ld s10, {s10}(t6)", + "sd t2, {s10}(t6)", + + "mv t3, s11", + "ld s11, {s11}(t6)", + "sd t3, {s11}(t6)", + + "mv t4, ra", + "ld ra, {ra}(t6)", + "sd t4, {ra}(t6)", + + "mv t5, tp", + "ld tp, {tp}(t6)", + "sd t5, {tp}(t6)", + + "mv a0, sp", + "ld sp, {sp}(t6)", + "sd a0, {sp}(t6)", + + "csrr t4, sstatus", + "ld t5, {sstatus}(t6)", + "sd t4, {sstatus}(t6)", + "ld t4, {sepc}(t6)", + + "ld gp, {gp}(t6)", + "ld a0, {a0}(t6)", + "ld a1, {a1}(t6)", + "ld a2, {a2}(t6)", + "ld a3, {a3}(t6)", + "ld a4, {a4}(t6)", + "ld a5, {a5}(t6)", + "ld a6, {a6}(t6)", + "ld a7, {a7}(t6)", + + "csrw sstatus, t5", + "csrw sepc, t4", + + "ld t0, {t0}(t6)", + "ld t1, {t1}(t6)", + "ld t2, {t2}(t6)", + "ld t3, {t3}(t6)", + "ld t4, {t4}(t6)", + "ld t5, {t5}(t6)", + "ld t6, {t6}(t6)", "sret", ra = const Registers::OFFSET_RA, sp = const Registers::OFFSET_SP, diff --git a/crates/eonix_hal/src/arch/riscv64/trap/mod.rs b/crates/eonix_hal/src/arch/riscv64/trap/mod.rs index 28689111..c62a2ff3 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/mod.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/mod.rs @@ -2,32 +2,24 @@ mod captured; mod default; mod trap_context; -use super::config::platform::virt::*; -use super::context::TaskContext; -use captured::{_captured_trap_entry, _captured_trap_return}; use core::arch::{global_asm, naked_asm}; use core::mem::{offset_of, size_of}; use core::num::NonZero; use core::ptr::NonNull; + +use captured::{_captured_trap_entry, _captured_trap_return}; use default::_default_trap_entry; -use eonix_hal_traits::{ - context::RawTaskContext, - trap::{IrqState as IrqStateTrait, TrapReturn}, -}; +use eonix_hal_traits::trap::{IrqState as IrqStateTrait, TrapReturn}; +use riscv::asm::sfence_vma_all; use riscv::register::sstatus::{self, Sstatus}; -use riscv::register::stvec::TrapMode; +use riscv::register::stvec::{self, Stvec, TrapMode}; use riscv::register::{scause, sepc, sscratch, stval}; -use riscv::{ - asm::sfence_vma_all, - register::stvec::{self, Stvec}, -}; use sbi::SbiError; - pub use trap_context::*; -impl TrapReturn for TrapContext { - type TaskContext = TaskContext; +use super::config::platform::virt::*; +impl TrapReturn for TrapContext { unsafe fn trap_return(&mut self) { let irq_states = disable_irqs_save(); @@ -42,14 +34,8 @@ impl TrapReturn for TrapContext { let old_trap_ctx = sscratch::read(); sscratch::write(&raw mut *self as usize); - let mut from_ctx = TaskContext::new(); - let mut to_ctx = TaskContext::new(); - to_ctx.set_program_counter(_captured_trap_return as usize); - to_ctx.set_stack_pointer(&raw mut from_ctx as usize); - to_ctx.set_interrupt_enabled(false); - unsafe { - TaskContext::switch(&mut from_ctx, &mut to_ctx); + _captured_trap_return(self); } sscratch::write(old_trap_ctx); @@ -57,6 +43,16 @@ impl TrapReturn for TrapContext { irq_states.restore(); } + + unsafe fn trap_return_noreturn(&mut self) -> ! { + disable_irqs(); + + unsafe { + _captured_trap_return(self); + } + + unreachable!("trap_return_noreturn should not return"); + } } fn setup_trap_handler(trap_entry_addr: usize) { diff --git a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs index ab4ca9a2..bb871be5 100644 --- a/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/riscv64/trap/trap_context.rs @@ -1,54 +1,53 @@ -use crate::{arch::time::set_next_timer, processor::CPU}; -use core::{arch::asm, mem::offset_of}; -use eonix_hal_traits::{ - fault::{Fault, PageFaultErrorCode}, - trap::{RawTrapContext, TrapType}, -}; +use core::arch::asm; +use core::mem::offset_of; + +use eonix_hal_traits::fault::{Fault, PageFaultErrorCode}; +use eonix_hal_traits::trap::{RawTrapContext, Stack, TrapType}; use eonix_mm::address::VAddr; -use riscv::{ - interrupt::{Exception, Interrupt, Trap}, - register::{ - scause::{self, Scause}, - sstatus::{self, Sstatus, FS, SPP}, - stval, - }, - ExceptionNumber, InterruptNumber, -}; +use riscv::interrupt::{Exception, Interrupt, Trap}; +use riscv::register::scause::{self, Scause}; +use riscv::register::sstatus::{self, Sstatus, FS, SPP}; +use riscv::register::stval; +use riscv::{ExceptionNumber, InterruptNumber}; + +use crate::arch::time::set_next_timer; +use crate::processor::CPU; #[repr(C)] #[derive(Default, Clone, Copy)] pub struct Registers { - tp: u64, + s0: u64, + s1: u64, + s2: u64, + s3: u64, + s4: u64, + s5: u64, + s6: u64, + s7: u64, + s8: u64, + s9: u64, + s10: u64, + s11: u64, ra: u64, + tp: u64, sp: u64, gp: u64, + a0: u64, a1: u64, a2: u64, a3: u64, a4: u64, - t1: u64, a5: u64, a6: u64, a7: u64, + t0: u64, + t1: u64, + t2: u64, t3: u64, t4: u64, t5: u64, - t2: u64, t6: u64, - s0: u64, - s1: u64, - s2: u64, - s3: u64, - s4: u64, - s5: u64, - s6: u64, - s7: u64, - s8: u64, - s9: u64, - s10: u64, - s11: u64, - t0: u64, } /// Saved CPU context when a trap (interrupt or exception) occurs on RISC-V 64. @@ -123,7 +122,7 @@ impl RawTrapContext for TrapContext { type FIrq = fn(handler: fn(irqno: usize)); type FTimer = fn(handler: fn()); - fn new() -> Self { + fn blank() -> Self { let mut sstatus = Sstatus::from_bits(0); sstatus.set_fs(FS::Initial); sstatus.set_sum(true); @@ -151,10 +150,13 @@ impl RawTrapContext for TrapContext { Interrupt::SupervisorExternal => TrapType::Irq { callback: |handler| { let mut cpu = CPU::local(); - match cpu.as_mut().interrupt.plic.claim_interrupt() { + match cpu.as_mut().interrupt.plic.claim_interrupt() + { None => {} Some(irqno) => { - cpu.interrupt.plic.complete_interrupt(irqno); + cpu.interrupt + .plic + .complete_interrupt(irqno); handler(irqno); } } @@ -172,18 +174,25 @@ impl RawTrapContext for TrapContext { | Exception::InstructionFault | Exception::LoadFault | Exception::StoreFault - | Exception::StoreMisaligned => TrapType::Fault(Fault::BadAccess), - Exception::IllegalInstruction => TrapType::Fault(Fault::InvalidOp), + | Exception::StoreMisaligned => { + TrapType::Fault(Fault::BadAccess) + } + Exception::IllegalInstruction => { + TrapType::Fault(Fault::InvalidOp) + } Exception::UserEnvCall => TrapType::Syscall { no: self.syscall_no(), args: self.syscall_args(), }, exception @ (Exception::InstructionPageFault | Exception::LoadPageFault - | Exception::StorePageFault) => TrapType::Fault(Fault::PageFault { - error_code: self.get_page_fault_error_code(exception), - address: VAddr::from(self.stval), - }), + | Exception::StorePageFault) => { + TrapType::Fault(Fault::PageFault { + error_code: self + .get_page_fault_error_code(exception), + address: VAddr::from(self.stval), + }) + } // breakpoint and supervisor env call _ => TrapType::Fault(Fault::Unknown(e)), } @@ -239,12 +248,8 @@ impl RawTrapContext for TrapContext { } fn set_user_call_frame<E>( - &mut self, - pc: usize, - sp: Option<usize>, - ra: Option<usize>, - args: &[usize], - _write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, + &mut self, pc: usize, sp: Option<usize>, ra: Option<usize>, + args: &[usize], _write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, ) -> Result<(), E> { self.set_program_counter(pc); @@ -271,11 +276,38 @@ impl RawTrapContext for TrapContext { Ok(()) } + + fn set_kernel_call_frame( + &mut self, pc: usize, sp: &impl Stack, ra: Option<usize>, + args: &[usize], + ) { + self.set_program_counter(pc); + self.set_stack_pointer(sp.get_bottom().addr()); + + if let Some(ra) = ra { + self.regs.ra = ra as u64; + } + + let arg_regs = [ + &mut self.regs.a0, + &mut self.regs.a1, + &mut self.regs.a2, + &mut self.regs.a3, + &mut self.regs.a4, + &mut self.regs.a5, + ]; + + for (&arg, reg) in args.iter().zip(arg_regs.into_iter()) { + *reg = arg as u64; + } + } } impl TrapContext { /// TODO: get PageFaultErrorCode also need check pagetable - fn get_page_fault_error_code(&self, exception: Exception) -> PageFaultErrorCode { + fn get_page_fault_error_code( + &self, exception: Exception, + ) -> PageFaultErrorCode { let mut error_code = PageFaultErrorCode::empty(); match exception { diff --git a/crates/eonix_hal/src/arch/x86_64/bootstrap.rs b/crates/eonix_hal/src/arch/x86_64/bootstrap.rs index f2637c50..87905d08 100644 --- a/crates/eonix_hal/src/arch/x86_64/bootstrap.rs +++ b/crates/eonix_hal/src/arch/x86_64/bootstrap.rs @@ -1,8 +1,9 @@ pub(crate) mod init; -use super::mm::{E820_MEM_MAP_DATA, PA_G, PA_NXE, PA_P, PA_PS, PA_RW}; use core::arch::{global_asm, naked_asm}; +use super::mm::{E820_MEM_MAP_DATA, PA_G, PA_NXE, PA_P, PA_PS, PA_RW}; + const KERNEL_IMAGE_PADDR: usize = 0x200000; const KERNEL_PML4: usize = 0x1000; const KERNEL_PDPT_PHYS_MAPPING: usize = 0x2000; @@ -171,7 +172,7 @@ global_asm!( .Lhalt: hlt jmp . - + # scratch %eax # return address should be of 2 bytes, and will be zero extended to 4 bytes .Lgo_32bit: @@ -408,17 +409,38 @@ global_asm!( mov ${KERNEL_PD_KIMAGE}, %esi call fill_pxe - # PDE 0xff0 + # PDE 0xfc0 ~ ? mov ${KERNEL_PD_KIMAGE}, %edi - lea 0xff0(%edi), %edi - mov ${KERNEL_PT_KIMAGE}, %esi # 0x104000 + mov ${KERNEL_IMAGE_PADDR}, %esi + lea 0xfc0(%edi), %edi + + mov ${KIMAGE_PAGES}, %ecx + shr $9, %ecx # 2M blocks + + or ${PA_PS}, %ebx + + 2: + test %ecx, %ecx + jz 2f + + call fill_pxe + lea 8(%edi), %edi + add $0x200000, %esi + dec %ecx + jmp 2b + + 2: + and $(~{PA_PS}), %ebx + push %esi + mov ${KERNEL_PT_KIMAGE}, %esi call fill_pxe # fill PT (kernel image) mov ${KERNEL_PT_KIMAGE}, %edi - mov ${KERNEL_IMAGE_PADDR}, %esi + pop %esi mov ${KIMAGE_PAGES}, %ecx + and $511, %ecx 2: call fill_pxe @@ -453,16 +475,18 @@ global_asm!( # %esi: page physical address # %edi: page x entry address fill_pxe: + push %eax lea (%ebx, %esi, 1), %eax mov %eax, (%edi) mov %edx, 4(%edi) + pop %eax ret .code64 2: jmp {start_64bit} - + .popsection "#, EARLY_GDT = sym EARLY_GDT, @@ -514,3 +538,14 @@ pub unsafe extern "C" fn start_64bit() { options(att_syntax) ) } + +pub fn shutdown() -> ! { + unsafe { + core::arch::asm!( + "mov $0x2000, %ax", + "mov $0x604, %dx", + "outw %ax, %dx", + options(att_syntax, nomem, noreturn) + ); + } +} diff --git a/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs b/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs index a3159a25..1945d2b6 100644 --- a/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs +++ b/crates/eonix_hal/src/arch/x86_64/bootstrap/init.rs @@ -1,30 +1,33 @@ -use crate::{ - arch::{ - bootstrap::{EARLY_GDT_DESCRIPTOR, KERNEL_PML4}, - cpu::{wrmsr, CPU}, - io::Port8, - mm::{ArchPhysAccess, GLOBAL_PAGE_TABLE, V_KERNEL_BSS_START}, - }, - bootstrap::BootStrapData, - mm::{ArchMemory, ArchPagingMode, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator}, -}; -use acpi::{platform::ProcessorState, AcpiHandler, AcpiTables, PhysicalMapping, PlatformInfo}; -use core::{ - alloc::Allocator, - arch::{asm, global_asm}, - cell::RefCell, - hint::spin_loop, - sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}, -}; +use core::alloc::Allocator; +use core::arch::{asm, global_asm}; +use core::cell::RefCell; +use core::hint::spin_loop; +use core::sync::atomic::{AtomicBool, AtomicPtr, AtomicUsize, Ordering}; + +use acpi::platform::ProcessorState; +use acpi::{AcpiHandler, AcpiTables, PhysicalMapping, PlatformInfo}; use eonix_hal_traits::mm::Memory; -use eonix_mm::{ - address::{Addr as _, PAddr, PRange, PhysAccess, VRange}, - page_table::{PageAttribute, PagingMode, PTE as _}, - paging::{Page, PageAccess, PageAlloc, PAGE_SIZE}, -}; +use eonix_mm::address::{Addr as _, PAddr, PRange, PhysAccess, VRange}; +use eonix_mm::page_table::{PageAttribute, PagingMode, PTE as _}; +use eonix_mm::paging::{Folio, FrameAlloc, PageAccess, PAGE_SIZE}; use eonix_percpu::PercpuArea; -static BSP_PAGE_ALLOC: AtomicPtr<RefCell<BasicPageAlloc>> = AtomicPtr::new(core::ptr::null_mut()); +use crate::arch::bootstrap::{EARLY_GDT_DESCRIPTOR, KERNEL_PML4}; +use crate::arch::cpu::{wrmsr, CPU}; +use crate::arch::io::Port8; +use crate::arch::mm::{ + with_global_page_table, ArchPhysAccess, PageAccessImpl, GLOBAL_PAGE_TABLE, + V_KERNEL_BSS_START, +}; +use crate::bootstrap::BootStrapData; +use crate::extern_symbol_value; +use crate::mm::{ + ArchMemory, BasicPageAlloc, BasicPageAllocRef, ScopedAllocator, +}; +use crate::processor::CPU_COUNT; + +static BSP_PAGE_ALLOC: AtomicPtr<RefCell<BasicPageAlloc>> = + AtomicPtr::new(core::ptr::null_mut()); static AP_COUNT: AtomicUsize = AtomicUsize::new(0); static AP_STACK: AtomicUsize = AtomicUsize::new(0); @@ -123,14 +126,19 @@ fn enable_sse() { } } -fn setup_cpu(alloc: impl PageAlloc) { +fn setup_cpu(alloc: impl FrameAlloc) { + CPU_COUNT.fetch_add(1, Ordering::AcqRel); + let mut percpu_area = PercpuArea::new(|layout| { // TODO: Use page size defined in `arch`. let page_count = layout.size().div_ceil(PAGE_SIZE); - let page = Page::alloc_at_least_in(page_count, alloc); + let folio = alloc.alloc_at_least(page_count).unwrap(); - let ptr = ArchPhysAccess::get_ptr_for_page(&page).cast(); - page.into_raw(); + let ptr = unsafe { + // TODO: safety + ArchPhysAccess::as_ptr(folio.start()) + }; + folio.into_raw(); ptr }); @@ -188,9 +196,7 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { impl AcpiHandler for Handler { unsafe fn map_physical_region<T>( - &self, - physical_address: usize, - size: usize, + &self, physical_address: usize, size: usize, ) -> PhysicalMapping<Self, T> { unsafe { PhysicalMapping::new( @@ -228,7 +234,7 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { let stack_range = { let page_alloc = BasicPageAllocRef::new(&page_alloc); - let ap_stack = Page::alloc_order_in(4, page_alloc); + let ap_stack = page_alloc.alloc_order(4).unwrap(); let stack_range = ap_stack.range(); ap_stack.into_raw(); @@ -236,7 +242,8 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { }; // SAFETY: All the APs can see the allocator work done before this point. - let old = BSP_PAGE_ALLOC.swap((&raw const *page_alloc) as *mut _, Ordering::Release); + let old = BSP_PAGE_ALLOC + .swap((&raw const *page_alloc) as *mut _, Ordering::Release); assert!( old.is_null(), "BSP_PAGE_ALLOC should be null before we release it" @@ -271,41 +278,39 @@ fn bootstrap_smp(alloc: impl Allocator, page_alloc: &RefCell<BasicPageAlloc>) { } pub extern "C" fn kernel_init() -> ! { - let global_page_table = &GLOBAL_PAGE_TABLE; - let paging_levels = ArchPagingMode::LEVELS; - enable_sse(); let real_allocator = RefCell::new(BasicPageAlloc::new()); let alloc = BasicPageAllocRef::new(&real_allocator); - unsafe extern "C" { - fn BSS_LENGTH(); - } + let bss_length = extern_symbol_value!(BSS_LENGTH); + let bss_range = VRange::from(V_KERNEL_BSS_START).grow(bss_length); for range in ArchMemory::free_ram() { real_allocator.borrow_mut().add_range(range); } // Map kernel BSS - for pte in global_page_table.iter_kernel_in( - VRange::from(V_KERNEL_BSS_START).grow(BSS_LENGTH as usize), - paging_levels, - &alloc, - ) { - let attr = PageAttribute::PRESENT - | PageAttribute::WRITE - | PageAttribute::READ - | PageAttribute::HUGE - | PageAttribute::GLOBAL; - - let page = Page::alloc_in(&alloc); - pte.set(page.into_raw(), attr.into()); - } + with_global_page_table(alloc.clone(), PageAccessImpl, |table| { + for pte in table.iter_kernel(bss_range) { + let attr = PageAttribute::PRESENT + | PageAttribute::WRITE + | PageAttribute::READ + | PageAttribute::HUGE + | PageAttribute::GLOBAL; + + let page = alloc.alloc().unwrap(); + pte.set(page.into_raw(), attr.into()); + } + }); unsafe { // SAFETY: We've just mapped the area with sufficient length. - core::ptr::write_bytes(V_KERNEL_BSS_START.addr() as *mut (), 0, BSS_LENGTH as usize); + core::ptr::write_bytes( + bss_range.start().addr() as *mut u8, + 0, + bss_length, + ); } setup_cpu(&alloc); @@ -329,12 +334,14 @@ pub extern "C" fn kernel_init() -> ! { } pub extern "C" fn ap_entry(stack_bottom: PAddr) -> ! { - let stack_range = PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); + let stack_range = + PRange::new(stack_bottom - (1 << 3) * PAGE_SIZE, stack_bottom); { // SAFETY: Acquire all the work done by the BSP and other APs. let alloc = loop { - let alloc = BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); + let alloc = + BSP_PAGE_ALLOC.swap(core::ptr::null_mut(), Ordering::AcqRel); if !alloc.is_null() { break alloc; diff --git a/crates/eonix_hal/src/arch/x86_64/context.rs b/crates/eonix_hal/src/arch/x86_64/context.rs deleted file mode 100644 index b6c28bf1..00000000 --- a/crates/eonix_hal/src/arch/x86_64/context.rs +++ /dev/null @@ -1,113 +0,0 @@ -use core::arch::naked_asm; -use eonix_hal_traits::context::RawTaskContext; - -/// Necessary hardware states of task for doing context switches. -#[repr(C)] -#[derive(Debug, Default)] -pub struct TaskContext { - r12: u64, - r13: u64, - r14: u64, - r15: u64, - rbx: u64, - rbp: u64, - rsp: u64, - rip: u64, // Should we save rip here? - rflags: u64, // Should we save rflags here? -} - -impl TaskContext { - /// Create a new task context with the given entry point and stack pointer. - /// The entry point is the function to be called when the task is scheduled. - /// The stack pointer is the address of the top of the stack. - /// The stack pointer should be aligned to 16 bytes. - pub const fn new() -> Self { - Self { - r12: 0, - r13: 0, - r14: 0, - r15: 0, - rbx: 0, - rbp: 0, - rsp: 0, - rip: 0, - rflags: 0x200, // IF = 1 by default. - } - } - - #[unsafe(naked)] - unsafe extern "C" fn do_call() -> ! { - naked_asm!( - "mov %r12, %rdi", - "push %rbp", // NULL return address. - "jmp *%rbx", - options(att_syntax), - ); - } -} - -impl RawTaskContext for TaskContext { - fn new() -> Self { - Self::new() - } - - fn set_program_counter(&mut self, pc: usize) { - self.rip = pc as u64; - } - - fn set_stack_pointer(&mut self, sp: usize) { - self.rsp = sp as u64; - } - - fn is_interrupt_enabled(&self) -> bool { - (self.rflags & 0x200) != 0 // IF = 1 - } - - fn set_interrupt_enabled(&mut self, is_enabled: bool) { - if is_enabled { - self.rflags |= 0x200; // IF = 1 - } else { - self.rflags &= !0x200; // IF = 0 - } - } - - fn call(&mut self, func: unsafe extern "C" fn(usize) -> !, arg: usize) { - self.set_program_counter(Self::do_call as _); - self.rbx = func as _; - self.r12 = arg as _; - self.rbp = 0; // NULL previous stack frame - } - - #[unsafe(naked)] - unsafe extern "C" fn switch(from: &mut Self, to: &mut Self) { - naked_asm!( - "pop %rax", - "pushf", - "pop %rcx", - "mov %r12, (%rdi)", - "mov %r13, 8(%rdi)", - "mov %r14, 16(%rdi)", - "mov %r15, 24(%rdi)", - "mov %rbx, 32(%rdi)", - "mov %rbp, 40(%rdi)", - "mov %rsp, 48(%rdi)", - "mov %rax, 56(%rdi)", - "mov %rcx, 64(%rdi)", - "", - "mov (%rsi), %r12", - "mov 8(%rsi), %r13", - "mov 16(%rsi), %r14", - "mov 24(%rsi), %r15", - "mov 32(%rsi), %rbx", - "mov 40(%rsi), %rbp", - "mov 48(%rsi), %rdi", // store next stack pointer - "mov 56(%rsi), %rax", - "mov 64(%rsi), %rcx", - "push %rcx", - "popf", - "xchg %rdi, %rsp", // switch to new stack - "jmp *%rax", - options(att_syntax), - ); - } -} diff --git a/crates/eonix_hal/src/arch/x86_64/cpu.rs b/crates/eonix_hal/src/arch/x86_64/cpu.rs index 26f4f1b6..951c8f32 100644 --- a/crates/eonix_hal/src/arch/x86_64/cpu.rs +++ b/crates/eonix_hal/src/arch/x86_64/cpu.rs @@ -1,13 +1,18 @@ -use super::gdt::{GDTEntry, GDT}; -use super::interrupt::InterruptControl; -use super::trap::TrapContext; use core::arch::asm; use core::marker::PhantomPinned; use core::mem::size_of; use core::pin::Pin; +use core::sync::atomic::AtomicUsize; + use eonix_preempt::PreemptGuard; use eonix_sync_base::LazyLock; +use super::gdt::{GDTEntry, GDT}; +use super::interrupt::InterruptControl; +use super::trap::TrapContext; + +pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(0); + #[eonix_percpu::define_percpu] static LOCAL_CPU: LazyLock<CPU> = LazyLock::new(CPU::new); @@ -33,17 +38,6 @@ pub(crate) struct TSS { _pinned: PhantomPinned, } -#[derive(Debug, Clone)] -pub enum UserTLS { - /// TODO: This is not used yet. - #[allow(dead_code)] - TLS64(u64), - TLS32 { - base: u64, - desc: GDTEntry, - }, -} - /// Architecture-specific cpu status data. pub struct CPU { cpuid: usize, @@ -52,22 +46,6 @@ pub struct CPU { interrupt: InterruptControl, } -impl UserTLS { - /// # Return - /// Returns the TLS descriptor and the index of the TLS segment. - pub fn new32(base: u32, limit: u32, is_limit_in_pages: bool) -> (Self, u32) { - let flags = if is_limit_in_pages { 0xc } else { 0x4 }; - - ( - Self::TLS32 { - base: base as u64, - desc: GDTEntry::new(base, limit, 0xf2, flags), - }, - 7, - ) - } -} - impl CPU { pub fn new() -> Self { let (interrupt_control, cpuid) = InterruptControl::new(); @@ -122,18 +100,14 @@ impl CPU { } } - pub fn set_tls32(self: Pin<&mut Self>, user_tls: &UserTLS) { - let UserTLS::TLS32 { desc, base } = user_tls else { - unimplemented!("TLS64 is not supported yet") - }; - + pub fn set_tls32(self: Pin<&mut Self>, desc: GDTEntry, base: u64) { unsafe { // SAFETY: We don't move the GDT object. - self.get_unchecked_mut().gdt.set_tls32(*desc); + self.get_unchecked_mut().gdt.set_tls32(desc); } const IA32_KERNEL_GS_BASE: u32 = 0xc0000102; - wrmsr(IA32_KERNEL_GS_BASE, *base); + wrmsr(IA32_KERNEL_GS_BASE, base); } pub fn cpuid(&self) -> usize { diff --git a/crates/eonix_hal/src/arch/x86_64/gdt.rs b/crates/eonix_hal/src/arch/x86_64/gdt.rs index 9db30793..a455d5b0 100644 --- a/crates/eonix_hal/src/arch/x86_64/gdt.rs +++ b/crates/eonix_hal/src/arch/x86_64/gdt.rs @@ -1,5 +1,7 @@ +use core::arch::asm; +use core::marker::PhantomPinned; + use super::cpu::TSS; -use core::{arch::asm, marker::PhantomPinned}; #[repr(transparent)] #[derive(Debug, Clone, Copy)] @@ -31,6 +33,14 @@ impl GDTEntry { GDTEntry(entry) } + pub const fn new_tls(base: u32, limit_in_bytes: u32) -> Self { + Self::new(base, limit_in_bytes, 0xf2, 0x4) + } + + pub const fn new_tls_page_limit(base: u32, limit_in_pages: u32) -> Self { + Self::new(base, limit_in_pages, 0xf2, 0xc) + } + pub const fn new_ldt(base: u64, limit: u32) -> [Self; 2] { let first = Self::new(base as u32, limit, 0x82, 0x0); let second = Self(base >> 32); @@ -46,7 +56,7 @@ impl GDTEntry { impl GDT { const LEN: usize = 10; - const TLS32_INDEX: usize = 7; + pub const TLS32_INDEX: usize = 7; const TSS_INDEX: usize = 8; pub fn new() -> Self { diff --git a/crates/eonix_hal/src/arch/x86_64/interrupt.rs b/crates/eonix_hal/src/arch/x86_64/interrupt.rs index 0ed44237..bc2692fd 100644 --- a/crates/eonix_hal/src/arch/x86_64/interrupt.rs +++ b/crates/eonix_hal/src/arch/x86_64/interrupt.rs @@ -1,5 +1,11 @@ +use core::arch::asm; +use core::marker::PhantomPinned; +use core::pin::Pin; +use core::ptr::NonNull; + use crate::arch::cpu::rdmsr; -use core::{arch::asm, marker::PhantomPinned, pin::Pin, ptr::NonNull}; +use crate::arch::trap::trap_stubs; +use crate::symbol_addr; #[repr(C)] #[derive(Clone, Copy)] @@ -109,7 +115,7 @@ impl InterruptControl { /// # Return /// Returns a tuple of InterruptControl and the cpu id of the current cpu. pub fn new() -> (Self, usize) { - let trap_stubs_base = super::trap::trap_stubs_start as usize; + let trap_stubs_base = symbol_addr!(trap_stubs); let idt = core::array::from_fn(|idx| match idx { 0..0x80 => IDTEntry::new(trap_stubs_base + 8 * idx, 0x08, 0x8e), @@ -121,7 +127,8 @@ impl InterruptControl { let apic_base = rdmsr(0x1b); assert_eq!(apic_base & 0x800, 0x800, "LAPIC not enabled"); - let apic_base = ((apic_base & !0xfff) + 0xffffff00_00000000) as *mut u32; + let apic_base = + ((apic_base & !0xfff) + 0xffffff00_00000000) as *mut u32; APICRegs { // TODO: A better way to convert to physical address base: NonNull::new(apic_base).expect("Invalid APIC base"), diff --git a/crates/eonix_hal/src/arch/x86_64/link.x b/crates/eonix_hal/src/arch/x86_64/link.x index 1fedd92e..d706a1b7 100644 --- a/crates/eonix_hal/src/arch/x86_64/link.x +++ b/crates/eonix_hal/src/arch/x86_64/link.x @@ -37,10 +37,12 @@ SECTIONS { SECTIONS { .vdso ALIGN(0x1000) : ALIGN(0x1000) { + VDSO_START = .; + KEEP(*(.vdso .vdso.*)); . = ALIGN(0x1000); - } > VDSO AT> REGION_TEXT + } > REGION_DATA AT> REGION_DATA VDSO_PADDR = LOADADDR(.vdso); } @@ -80,6 +82,30 @@ SECTIONS { RAW_SYSCALL_HANDLERS_SIZE = ABSOLUTE(__raw_syscall_handlers_end - __raw_syscall_handlers_start); } > REGION_RODATA + + .rodata.late_init : + { + . = ALIGN(16); + __raw_late_init_start = .; + + LATE_INIT_FUNCTIONS = .; + KEEP(*(.late_init)); + + __raw_late_init_end = .; + + __raw_late_init_async_start = .; + + LATE_INIT_ASYNC_FUNCTIONS = .; + KEEP(*(.late_init_async)); + + __raw_late_init_async_end = .; + + LATE_INIT_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_end - __raw_late_init_start) / 8); + + LATE_INIT_ASYNC_FUNCTION_COUNT = + ABSOLUTE((__raw_late_init_async_end - __raw_late_init_async_start) / 8); + } > REGION_RODATA } INSERT AFTER .rodata; @@ -89,6 +115,10 @@ SECTIONS { __spercpu = .; QUAD(0); /* Reserved for x86 percpu pointer */ + QUAD(0); /* Reserved for x86 percpu capturer trap context */ + + QUAD(0); /* Reserved for x86 percpu capturer scratch */ + QUAD(0); /* Reserved for x86 percpu capturer scratch */ . = ALIGN(16); diff --git a/crates/eonix_hal/src/arch/x86_64/memory.x b/crates/eonix_hal/src/arch/x86_64/memory.x index 63f05753..67fdb7b1 100644 --- a/crates/eonix_hal/src/arch/x86_64/memory.x +++ b/crates/eonix_hal/src/arch/x86_64/memory.x @@ -2,7 +2,7 @@ MEMORY { LOWMEM : org = 0x0000000000000000, len = 1M VDSO : org = 0x00007f0000000000, len = 4K KBSS : org = 0xffffffffc0200000, len = 2M - KIMAGE : org = 0xffffffffffc00000, len = 2M + KIMAGE : org = 0xffffffffff000000, len = 14M } REGION_ALIAS("REGION_TEXT", KIMAGE); diff --git a/crates/eonix_hal/src/arch/x86_64/mm.rs b/crates/eonix_hal/src/arch/x86_64/mm.rs index 9dcae396..11bc2996 100644 --- a/crates/eonix_hal/src/arch/x86_64/mm.rs +++ b/crates/eonix_hal/src/arch/x86_64/mm.rs @@ -1,15 +1,19 @@ -use crate::traits::mm::Memory; -use core::{arch::asm, marker::PhantomData, ptr::NonNull}; -use eonix_mm::{ - address::{Addr as _, AddrOps as _, PAddr, PRange, PhysAccess, VAddr}, - page_table::{ - PageAttribute, PageTable, PageTableLevel, PagingMode, RawAttribute, RawPageTable, - TableAttribute, PTE, - }, - paging::{NoAlloc, Page, PageBlock, PAGE_SIZE, PFN}, +use core::arch::asm; +use core::marker::PhantomData; +use core::ptr::NonNull; + +use eonix_mm::address::{ + Addr as _, AddrOps as _, PAddr, PRange, PhysAccess, VAddr, }; +use eonix_mm::page_table::{ + PageAttribute, PageTable, PageTableAlloc, PageTableLevel, PagingMode, + RawAttribute, RawPageTable, TableAttribute, PTE, +}; +use eonix_mm::paging::{BasicFolio, PageAccess, PageBlock, PAGE_SIZE, PFN}; use eonix_sync_base::LazyLock; +use crate::traits::mm::Memory; + pub const PA_P: u64 = 0x001; pub const PA_RW: u64 = 0x002; pub const PA_US: u64 = 0x004; @@ -31,13 +35,7 @@ pub const P_KIMAGE_START: PAddr = PAddr::from_val(0x200000); pub const V_KERNEL_BSS_START: VAddr = VAddr::from(0xffffffffc0200000); const KERNEL_PML4_PFN: PFN = PFN::from_val(0x1000 >> 12); - -pub static GLOBAL_PAGE_TABLE: LazyLock<PageTable<ArchPagingMode, NoAlloc, ArchPhysAccess>> = - LazyLock::new(|| unsafe { - Page::with_raw(KERNEL_PML4_PFN, |root_table_page| { - PageTable::with_root_table(root_table_page.clone()) - }) - }); +pub const GLOBAL_PAGE_TABLE: BasicFolio = BasicFolio::new(KERNEL_PML4_PFN, 0); #[repr(transparent)] pub struct PTE64(u64); @@ -51,6 +49,9 @@ pub struct PagingMode4Levels; pub struct ArchPhysAccess; +#[derive(Clone)] +pub struct PageAccessImpl; + pub struct ArchMemory; #[repr(C)] @@ -119,6 +120,9 @@ impl<'a> RawPageTable<'a> for RawPageTable4Levels<'a> { } } +unsafe impl Send for RawPageTable4Levels<'_> {} +unsafe impl Sync for RawPageTable4Levels<'_> {} + impl RawAttribute for PageAttribute64 { fn null() -> Self { Self(0) @@ -273,6 +277,12 @@ impl PhysAccess for ArchPhysAccess { } } +impl PageAccess for PageAccessImpl { + unsafe fn get_ptr_for_pfn(&self, pfn: PFN) -> NonNull<PageBlock> { + unsafe { ArchPhysAccess::as_ptr(PAddr::from(pfn)) } + } +} + impl E820MemMapEntry { const ENTRY_FREE: u32 = 1; // const ENTRY_USED: u32 = 2; @@ -420,3 +430,18 @@ pub fn set_root_page_table_pfn(pfn: PFN) { ); } } + +pub fn with_global_page_table<A, X>( + alloc: A, access: X, + func: impl FnOnce(&mut PageTable<ArchPagingMode, A, X>), +) where + A: PageTableAlloc<Folio = BasicFolio>, + X: PageAccess, +{ + let mut global_page_table = + PageTable::new(GLOBAL_PAGE_TABLE.clone(), alloc, access); + + func(&mut global_page_table); + + core::mem::forget(global_page_table); +} diff --git a/crates/eonix_hal/src/arch/x86_64/mod.rs b/crates/eonix_hal/src/arch/x86_64/mod.rs index 177576fc..a7f760dc 100644 --- a/crates/eonix_hal/src/arch/x86_64/mod.rs +++ b/crates/eonix_hal/src/arch/x86_64/mod.rs @@ -1,5 +1,4 @@ pub mod bootstrap; -pub mod context; pub mod cpu; pub mod fence; pub mod fpu; diff --git a/crates/eonix_hal/src/arch/x86_64/trap.rs b/crates/eonix_hal/src/arch/x86_64/trap.rs deleted file mode 100644 index 6a47a753..00000000 --- a/crates/eonix_hal/src/arch/x86_64/trap.rs +++ /dev/null @@ -1,392 +0,0 @@ -mod trap_context; - -use super::context::TaskContext; -use core::arch::{asm, global_asm, naked_asm}; -use eonix_hal_traits::{ - context::RawTaskContext, - trap::{IrqState as IrqStateTrait, TrapReturn}, -}; - -pub use trap_context::TrapContext; - -unsafe extern "C" { - fn _default_trap_handler(trap_context: &mut TrapContext); - pub fn trap_stubs_start(); - fn _raw_trap_return(); -} - -#[eonix_percpu::define_percpu] -static TRAP_HANDLER: unsafe extern "C" fn() = default_trap_handler; - -#[eonix_percpu::define_percpu] -static CAPTURER_CONTEXT: TaskContext = TaskContext::new(); - -/// This value will never be used. -static mut DIRTY_TRAP_CONTEXT: TaskContext = TaskContext::new(); - -/// State of the interrupt flag. -pub struct IrqState(u64); - -global_asm!( - r" - .set RAX, 0x00 - .set RBX, 0x08 - .set RCX, 0x10 - .set RDX, 0x18 - .set RDI, 0x20 - .set RSI, 0x28 - .set R8, 0x30 - .set R9, 0x38 - .set R10, 0x40 - .set R11, 0x48 - .set R12, 0x50 - .set R13, 0x58 - .set R14, 0x60 - .set R15, 0x68 - .set RBP, 0x70 - .set INT_NO, 0x78 - .set ERRCODE, 0x80 - .set RIP, 0x88 - .set CS, 0x90 - .set FLAGS, 0x98 - .set RSP, 0xa0 - .set SS, 0xa8 - - .macro cfi_all_same_value - .cfi_same_value %rax - .cfi_same_value %rbx - .cfi_same_value %rcx - .cfi_same_value %rdx - .cfi_same_value %rdi - .cfi_same_value %rsi - .cfi_same_value %r8 - .cfi_same_value %r9 - .cfi_same_value %r10 - .cfi_same_value %r11 - .cfi_same_value %r12 - .cfi_same_value %r13 - .cfi_same_value %r14 - .cfi_same_value %r15 - .cfi_same_value %rbp - .endm - - .globl {trap_stubs_start} - {trap_stubs_start}: - .altmacro - .macro build_isr_no_err name - .align 8 - .globl ISR\name - .type ISR\name @function - ISR\name: - .cfi_startproc - .cfi_signal_frame - .cfi_def_cfa_offset 0x08 - .cfi_offset %rsp, 0x10 - - cfi_all_same_value - - push %rbp # push placeholder for error code - .cfi_def_cfa_offset 0x10 - - call _raw_trap_entry - .cfi_endproc - .endm - - .altmacro - .macro build_isr_err name - .align 8 - .globl ISR\name - .type ISR\name @function - ISR\name: - .cfi_startproc - .cfi_signal_frame - .cfi_def_cfa_offset 0x10 - .cfi_offset %rsp, 0x10 - - cfi_all_same_value - - call _raw_trap_entry - .cfi_endproc - .endm - - build_isr_no_err 0 - build_isr_no_err 1 - build_isr_no_err 2 - build_isr_no_err 3 - build_isr_no_err 4 - build_isr_no_err 5 - build_isr_no_err 6 - build_isr_no_err 7 - build_isr_err 8 - build_isr_no_err 9 - build_isr_err 10 - build_isr_err 11 - build_isr_err 12 - build_isr_err 13 - build_isr_err 14 - build_isr_no_err 15 - build_isr_no_err 16 - build_isr_err 17 - build_isr_no_err 18 - build_isr_no_err 19 - build_isr_no_err 20 - build_isr_err 21 - build_isr_no_err 22 - build_isr_no_err 23 - build_isr_no_err 24 - build_isr_no_err 25 - build_isr_no_err 26 - build_isr_no_err 27 - build_isr_no_err 28 - build_isr_err 29 - build_isr_err 30 - build_isr_no_err 31 - - .set i, 32 - .rept 0x80+1 - build_isr_no_err %i - .set i, i+1 - .endr - - .globl _raw_trap_entry - .type _raw_trap_entry @function - _raw_trap_entry: - .cfi_startproc - .cfi_signal_frame - .cfi_def_cfa %rsp, 0x18 - .cfi_offset %rsp, 0x10 - - cfi_all_same_value - - sub $0x78, %rsp - .cfi_def_cfa_offset CS - - mov %rax, RAX(%rsp) - .cfi_rel_offset %rax, RAX - mov %rbx, RBX(%rsp) - .cfi_rel_offset %rbx, RBX - mov %rcx, RCX(%rsp) - .cfi_rel_offset %rcx, RCX - mov %rdx, RDX(%rsp) - .cfi_rel_offset %rdx, RDX - mov %rdi, RDI(%rsp) - .cfi_rel_offset %rdi, RDI - mov %rsi, RSI(%rsp) - .cfi_rel_offset %rsi, RSI - mov %r8, R8(%rsp) - .cfi_rel_offset %r8, R8 - mov %r9, R9(%rsp) - .cfi_rel_offset %r9, R9 - mov %r10, R10(%rsp) - .cfi_rel_offset %r10, R10 - mov %r11, R11(%rsp) - .cfi_rel_offset %r11, R11 - mov %r12, R12(%rsp) - .cfi_rel_offset %r12, R12 - mov %r13, R13(%rsp) - .cfi_rel_offset %r13, R13 - mov %r14, R14(%rsp) - .cfi_rel_offset %r14, R14 - mov %r15, R15(%rsp) - .cfi_rel_offset %r15, R15 - mov %rbp, RBP(%rsp) - .cfi_rel_offset %rbp, RBP - - mov INT_NO(%rsp), %rcx - sub ${trap_stubs_start}, %rcx - shr $3, %rcx - mov %rcx, INT_NO(%rsp) - - cmpq $0x08, CS(%rsp) - je 2f - swapgs - - 2: - mov %gs:0, %rcx - add ${handler}, %rcx - mov (%rcx), %rcx - - jmp *%rcx - .cfi_endproc - - _raw_trap_return: - .cfi_startproc - .cfi_def_cfa %rsp, CS - .cfi_rel_offset %rax, RAX - .cfi_rel_offset %rbx, RBX - .cfi_rel_offset %rcx, RCX - .cfi_rel_offset %rdx, RDX - .cfi_rel_offset %rdi, RDI - .cfi_rel_offset %rsi, RSI - .cfi_rel_offset %r8, R8 - .cfi_rel_offset %r9, R9 - .cfi_rel_offset %r10, R10 - .cfi_rel_offset %r11, R11 - .cfi_rel_offset %r12, R12 - .cfi_rel_offset %r13, R13 - .cfi_rel_offset %r14, R14 - .cfi_rel_offset %r15, R15 - .cfi_rel_offset %rbp, RBP - .cfi_rel_offset %rsp, RSP - - mov RAX(%rsp), %rax - .cfi_restore %rax - mov RBX(%rsp), %rbx - .cfi_restore %rbx - mov RCX(%rsp), %rcx - .cfi_restore %rcx - mov RDX(%rsp), %rdx - .cfi_restore %rdx - mov RDI(%rsp), %rdi - .cfi_restore %rdi - mov RSI(%rsp), %rsi - .cfi_restore %rsi - mov R8(%rsp), %r8 - .cfi_restore %r8 - mov R9(%rsp), %r9 - .cfi_restore %r9 - mov R10(%rsp), %r10 - .cfi_restore %r10 - mov R11(%rsp), %r11 - .cfi_restore %r11 - mov R12(%rsp), %r12 - .cfi_restore %r12 - mov R13(%rsp), %r13 - .cfi_restore %r13 - mov R14(%rsp), %r14 - .cfi_restore %r14 - mov R15(%rsp), %r15 - .cfi_restore %r15 - mov RBP(%rsp), %rbp - .cfi_restore %rbp - - cmpq $0x08, CS(%rsp) - je 2f - swapgs - - 2: - lea RIP(%rsp), %rsp - .cfi_def_cfa %rsp, 0x08 - .cfi_offset %rsp, 0x10 - - iretq - .cfi_endproc - ", - trap_stubs_start = sym trap_stubs_start, - handler = sym _percpu_inner_TRAP_HANDLER, - options(att_syntax), -); - -/// Default handler handles the trap on the current stack and returns -/// to the context before interrut. -#[unsafe(naked)] -unsafe extern "C" fn default_trap_handler() { - naked_asm!( - ".cfi_startproc", - "mov %rsp, %rbx", - ".cfi_def_cfa_register %rbx", - "", - "and $~0xf, %rsp", - "", - "mov %rbx, %rdi", - "call {handle_trap}", - "", - "mov %rbx, %rsp", - ".cfi_def_cfa_register %rsp", - "", - "jmp {trap_return}", - ".cfi_endproc", - handle_trap = sym _default_trap_handler, - trap_return = sym _raw_trap_return, - options(att_syntax), - ); -} - -#[unsafe(naked)] -unsafe extern "C" fn captured_trap_handler() { - naked_asm!( - "mov ${from_context}, %rdi", - "mov %gs:0, %rsi", - "add ${to_context}, %rsi", - "", - "mov %rdi, %rsp", // We need a temporary stack to use `switch()`. - "", - "jmp {switch}", - from_context = sym DIRTY_TRAP_CONTEXT, - to_context = sym _percpu_inner_CAPTURER_CONTEXT, - switch = sym TaskContext::switch, - options(att_syntax), - ); -} - -#[unsafe(naked)] -unsafe extern "C" fn captured_trap_return(trap_context: usize) -> ! { - naked_asm!( - "jmp {trap_return}", - trap_return = sym _raw_trap_return, - options(att_syntax), - ); -} - -impl TrapReturn for TrapContext { - type TaskContext = TaskContext; - - unsafe fn trap_return(&mut self) { - let irq_states = disable_irqs_save(); - let old_handler = TRAP_HANDLER.swap(captured_trap_handler); - - let mut to_ctx = TaskContext::new(); - to_ctx.set_program_counter(captured_trap_return as _); - to_ctx.set_stack_pointer(&raw mut *self as usize); - to_ctx.set_interrupt_enabled(false); - - unsafe { - TaskContext::switch(CAPTURER_CONTEXT.as_mut(), &mut to_ctx); - } - - TRAP_HANDLER.set(old_handler); - irq_states.restore(); - } -} - -impl IrqStateTrait for IrqState { - fn restore(self) { - let Self(state) = self; - - unsafe { - asm!( - "push {state}", - "popf", - state = in(reg) state, - options(att_syntax, nomem) - ); - } - } -} - -pub fn enable_irqs() { - unsafe { - asm!("sti", options(att_syntax, nomem, nostack)); - } -} - -pub fn disable_irqs() { - unsafe { - asm!("cli", options(att_syntax, nomem, nostack)); - } -} - -pub fn disable_irqs_save() -> IrqState { - let state: u64; - unsafe { - asm!( - "pushf", - "pop {state}", - "cli", - state = out(reg) state, - options(att_syntax, nomem) - ); - } - - IrqState(state) -} diff --git a/crates/eonix_hal/src/arch/x86_64/trap/mod.rs b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs new file mode 100644 index 00000000..008d996e --- /dev/null +++ b/crates/eonix_hal/src/arch/x86_64/trap/mod.rs @@ -0,0 +1,634 @@ +mod trap_context; + +use core::arch::{asm, global_asm, naked_asm}; + +use eonix_hal_traits::trap::{ + IrqState as IrqStateTrait, RawTrapContext, TrapReturn, +}; +pub use trap_context::TrapContext; + +use super::cpu::CPU; + +unsafe extern "C" { + /// Default handler handles the trap on the current stack and returns + /// to the context before interrut. + fn _default_trap_handler(trap_context: &mut TrapContext); +} + +/// State of the interrupt flag. +pub struct IrqState(u64); + +macro_rules! cfi_all_same_value { + () => { + " + .cfi_same_value %rax + .cfi_same_value %rbx + .cfi_same_value %rcx + .cfi_same_value %rdx + .cfi_same_value %rdi + .cfi_same_value %rsi + .cfi_same_value %r8 + .cfi_same_value %r9 + .cfi_same_value %r10 + .cfi_same_value %r11 + .cfi_same_value %r12 + .cfi_same_value %r13 + .cfi_same_value %r14 + .cfi_same_value %r15 + .cfi_same_value %rbp + " + }; +} + +#[unsafe(naked)] +pub unsafe extern "C" fn trap_stubs() { + naked_asm!( + " + .altmacro + .macro build_isr_no_err name + .align 8 + .globl ISR\\name + .type ISR\\name @function + ISR\\name: + .cfi_startproc + .cfi_signal_frame + .cfi_def_cfa_offset 0x08 + .cfi_offset %rsp, 0x10 + ", + cfi_all_same_value!(), + " + + push %rbp # push placeholder for error code + .cfi_def_cfa_offset 0x10 + + call {entry} + .cfi_endproc + .endm + + .altmacro + .macro build_isr_err name + .align 8 + .globl ISR\\name + .type ISR\\name @function + ISR\\name: + .cfi_startproc + .cfi_signal_frame + .cfi_def_cfa_offset 0x10 + .cfi_offset %rsp, 0x10 + ", + cfi_all_same_value!(), + " + + call {entry} + .cfi_endproc + .endm + + build_isr_no_err 0 + build_isr_no_err 1 + build_isr_no_err 2 + build_isr_no_err 3 + build_isr_no_err 4 + build_isr_no_err 5 + build_isr_no_err 6 + build_isr_no_err 7 + build_isr_err 8 + build_isr_no_err 9 + build_isr_err 10 + build_isr_err 11 + build_isr_err 12 + build_isr_err 13 + build_isr_err 14 + build_isr_no_err 15 + build_isr_no_err 16 + build_isr_err 17 + build_isr_no_err 18 + build_isr_no_err 19 + build_isr_no_err 20 + build_isr_err 21 + build_isr_no_err 22 + build_isr_no_err 23 + build_isr_no_err 24 + build_isr_no_err 25 + build_isr_no_err 26 + build_isr_no_err 27 + build_isr_no_err 28 + build_isr_err 29 + build_isr_err 30 + build_isr_no_err 31 + + .set i, 32 + .rept 0x80+1 + build_isr_no_err %i + .set i, i+1 + .endr + ", + entry = sym raw_trap_entry, + options(att_syntax), + ) +} + +/// Offset of the capturer trap context in the percpu area. +const OFFSET_CAPTURER: usize = 8; + +#[unsafe(naked)] +unsafe extern "C" fn raw_trap_entry() { + naked_asm!( + ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x18", + ".cfi_offset %rsp, 0x10", + cfi_all_same_value!(), + "", + "cmpq $0x08, 0x18(%rsp)", + "je 2f", + "swapgs", + "", + "2:", + "subq ${trap_stubs}, (%rsp)", + "shrq $3, (%rsp)", + "", + "cmpq $0, %gs:{offset_capturer}", + "je {default_entry}", + "", + "cmpq $0x08, 0x18(%rsp)", + "je {captured_kernel_entry}", + "jmp {captured_user_entry}", + ".cfi_endproc", + trap_stubs = sym trap_stubs, + default_entry = sym default_trap_entry, + captured_kernel_entry = sym captured_trap_entry_kernel, + captured_user_entry = sym captured_trap_entry_user, + offset_capturer = const OFFSET_CAPTURER, + options(att_syntax), + ) +} + +#[unsafe(naked)] +unsafe extern "C" fn default_trap_entry() { + naked_asm!( + ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x18", + ".cfi_offset %rsp, 0x10", + cfi_all_same_value!(), + "", + "sub ${INT_NO}, %rsp", + ".cfi_def_cfa_offset {CS}", + "", + "mov %rcx, {RCX}(%rsp)", + ".cfi_rel_offset %rcx, {RCX}", + "mov %rdx, {RDX}(%rsp)", + ".cfi_rel_offset %rdx, {RDX}", + "mov %rdi, {RDI}(%rsp)", + ".cfi_rel_offset %rdi, {RDI}", + "mov %rsi, {RSI}(%rsp)", + ".cfi_rel_offset %rsi, {RSI}", + "mov %r8, {R8}(%rsp)", + ".cfi_rel_offset %r8, {R8}", + "mov %r9, {R9}(%rsp)", + ".cfi_rel_offset %r9, {R9}", + "mov %r10, {R10}(%rsp)", + ".cfi_rel_offset %r10, {R10}", + "mov %r11, {R11}(%rsp)", + ".cfi_rel_offset %r11, {R11}", + "mov %rbx, {RBX}(%rsp)", + ".cfi_rel_offset %rbx, {RBX}", + "mov %r12, {R12}(%rsp)", + ".cfi_rel_offset %r12, {R12}", + "", + "mov %rax, %r12", + ".cfi_register %rax, %r12", + "mov %rsp, %rbx", + ".cfi_def_cfa_register %rbx", + "", + "and $-0x10, %rsp", + "mov %rbx, %rdi", + "", + "call {default_entry}", + "", + "mov %rbx, %rsp", + ".cfi_def_cfa_register %rsp", + "mov %r12, %rax", + ".cfi_restore %rax", + "", + "mov {RCX}(%rsp), %rcx", + ".cfi_restore %rcx", + "mov {RDX}(%rsp), %rdx", + ".cfi_restore %rdx", + "mov {RDI}(%rsp), %rdi", + ".cfi_restore %rdi", + "mov {RSI}(%rsp), %rsi", + ".cfi_restore %rsi", + "mov {R8}(%rsp), %r8", + ".cfi_restore %r8", + "mov {R9}(%rsp), %r9", + ".cfi_restore %r9", + "mov {R10}(%rsp), %r10", + ".cfi_restore %r10", + "mov {R11}(%rsp), %r11", + ".cfi_restore %r11", + "mov {RBX}(%rsp), %rbx", + ".cfi_restore %rbx", + "mov {R12}(%rsp), %r12", + ".cfi_restore %r12", + "", + "cmpq $0x08, {CS}(%rsp)", + "je 2f", + "swapgs", + "", + "2:", + "lea {RIP}(%rsp), %rsp", + ".cfi_def_cfa %rsp, 0x08", + ".cfi_offset %rsp, 0x10", + "", + "iretq", + ".cfi_endproc", + default_entry = sym _default_trap_handler, + RBX = const TrapContext::OFFSET_RBX, + RCX = const TrapContext::OFFSET_RCX, + RDX = const TrapContext::OFFSET_RDX, + RDI = const TrapContext::OFFSET_RDI, + RSI = const TrapContext::OFFSET_RSI, + R8 = const TrapContext::OFFSET_R8, + R9 = const TrapContext::OFFSET_R9, + R10 = const TrapContext::OFFSET_R10, + R11 = const TrapContext::OFFSET_R11, + R12 = const TrapContext::OFFSET_R12, + INT_NO = const TrapContext::OFFSET_INT_NO, + RIP = const TrapContext::OFFSET_RIP, + CS = const TrapContext::OFFSET_CS, + options(att_syntax), + ) +} + +#[unsafe(naked)] +unsafe extern "C" fn captured_trap_entry_kernel() { + naked_asm!( + ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x18", + ".cfi_offset %rsp, 0x10", + cfi_all_same_value!(), + "", + "mov %rsi, %gs:0x10", + ".cfi_undefined %rsi", + "mov %rsp, %rsi", + ".cfi_def_cfa_register %rsi", + ".cfi_register %rsp, %rsi", + + "mov %gs:0x08, %rsp", + // Save and load registers. + "mov %rcx, {RCX}(%rsp)", + ".cfi_rel_offset %rcx, {RCX}", + "", + "mov %gs:0x10, %rcx", + ".cfi_register %rsi, %rcx", + "", + "mov %rdx, {RDX}(%rsp)", + ".cfi_rel_offset %rdx, {RDX}", + "mov %rdi, {RDI}(%rsp)", + ".cfi_rel_offset %rdi, {RDI}", + "mov %rcx, {RSI}(%rsp)", + ".cfi_rel_offset %rsi, {RSI}", + "mov %r8, {R8}(%rsp)", + ".cfi_rel_offset %r8, {R8}", + "mov %r9, {R9}(%rsp)", + ".cfi_rel_offset %r9, {R9}", + "mov %r10, {R10}(%rsp)", + ".cfi_rel_offset %r10, {R10}", + "mov %r11, {R11}(%rsp)", + ".cfi_rel_offset %r11, {R11}", + "", + "xchg %rax, {RAX}(%rsp)", + ".cfi_rel_offset %rax, {RAX}", + "xchg %rbx, {RBX}(%rsp)", + ".cfi_rel_offset %rbx, {RBX}", + "xchg %r12, {R12}(%rsp)", + ".cfi_rel_offset %r12, {R12}", + "xchg %r13, {R13}(%rsp)", + ".cfi_rel_offset %r13, {R13}", + "xchg %r14, {R14}(%rsp)", + ".cfi_rel_offset %r14, {R14}", + "xchg %r15, {R15}(%rsp)", + ".cfi_rel_offset %r15, {R15}", + "xchg %rbp, {RBP}(%rsp)", + ".cfi_rel_offset %rbp, {RBP}", + "", + "lea {INT_NO}(%rsp), %rdi", + "mov $7, %rcx", + "cld", + "rep movsq", + "", + "mov %rax, %rsp", + ".cfi_def_cfa %rsp, 0x10", + ".cfi_undefined %rax", + ".cfi_restore %rbx", + ".cfi_undefined %rcx", + ".cfi_undefined %rdx", + ".cfi_undefined %rdi", + ".cfi_undefined %rsi", + ".cfi_undefined %r8", + ".cfi_undefined %r9", + ".cfi_undefined %r10", + ".cfi_undefined %r11", + ".cfi_restore %r12", + ".cfi_restore %r13", + ".cfi_restore %r14", + ".cfi_restore %r15", + ".cfi_restore %rbp", + "", + "popf", + ".cfi_def_cfa_offset 0x08", + "", + "ret", + ".cfi_endproc", + RAX = const TrapContext::OFFSET_RAX, + RBX = const TrapContext::OFFSET_RBX, + RCX = const TrapContext::OFFSET_RCX, + RDX = const TrapContext::OFFSET_RDX, + RDI = const TrapContext::OFFSET_RDI, + RSI = const TrapContext::OFFSET_RSI, + R8 = const TrapContext::OFFSET_R8, + R9 = const TrapContext::OFFSET_R9, + R10 = const TrapContext::OFFSET_R10, + R11 = const TrapContext::OFFSET_R11, + R12 = const TrapContext::OFFSET_R12, + R13 = const TrapContext::OFFSET_R13, + R14 = const TrapContext::OFFSET_R14, + R15 = const TrapContext::OFFSET_R15, + RBP = const TrapContext::OFFSET_RBP, + INT_NO = const TrapContext::OFFSET_INT_NO, + options(att_syntax), + ) +} + +#[unsafe(naked)] +unsafe extern "C" fn captured_trap_entry_user() { + naked_asm!( + ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x18", + ".cfi_offset %rsp, 0x10", + cfi_all_same_value!(), + "", + "sub ${INT_NO}, %rsp", + ".cfi_def_cfa_offset {CS}", + "", + // Save and load registers. + "mov %rcx, {RCX}(%rsp)", + ".cfi_rel_offset %rcx, {RCX}", + "mov %rdx, {RDX}(%rsp)", + ".cfi_rel_offset %rdx, {RDX}", + "mov %rdi, {RDI}(%rsp)", + ".cfi_rel_offset %rdi, {RDI}", + "mov %rsi, {RSI}(%rsp)", + ".cfi_rel_offset %rsi, {RSI}", + "mov %r8, {R8}(%rsp)", + ".cfi_rel_offset %r8, {R8}", + "mov %r9, {R9}(%rsp)", + ".cfi_rel_offset %r9, {R9}", + "mov %r10, {R10}(%rsp)", + ".cfi_rel_offset %r10, {R10}", + "mov %r11, {R11}(%rsp)", + ".cfi_rel_offset %r11, {R11}", + "", + "xchg %rax, {RAX}(%rsp)", + ".cfi_rel_offset %rax, {RAX}", + "xchg %rbx, {RBX}(%rsp)", + ".cfi_rel_offset %rbx, {RBX}", + "xchg %r12, {R12}(%rsp)", + ".cfi_rel_offset %r12, {R12}", + "xchg %r13, {R13}(%rsp)", + ".cfi_rel_offset %r13, {R13}", + "xchg %r14, {R14}(%rsp)", + ".cfi_rel_offset %r14, {R14}", + "xchg %r15, {R15}(%rsp)", + ".cfi_rel_offset %r15, {R15}", + "xchg %rbp, {RBP}(%rsp)", + ".cfi_rel_offset %rbp, {RBP}", + "", + "mov %rax, %rsp", + ".cfi_def_cfa %rsp, 0x10", + ".cfi_undefined %rax", + ".cfi_restore %rbx", + ".cfi_undefined %rcx", + ".cfi_undefined %rdx", + ".cfi_undefined %rdi", + ".cfi_undefined %rsi", + ".cfi_undefined %r8", + ".cfi_undefined %r9", + ".cfi_undefined %r10", + ".cfi_undefined %r11", + ".cfi_restore %r12", + ".cfi_restore %r13", + ".cfi_restore %r14", + ".cfi_restore %r15", + ".cfi_restore %rbp", + "", + "popf", + ".cfi_def_cfa_offset 0x08", + "", + "ret", + ".cfi_endproc", + RAX = const TrapContext::OFFSET_RAX, + RBX = const TrapContext::OFFSET_RBX, + RCX = const TrapContext::OFFSET_RCX, + RDX = const TrapContext::OFFSET_RDX, + RDI = const TrapContext::OFFSET_RDI, + RSI = const TrapContext::OFFSET_RSI, + R8 = const TrapContext::OFFSET_R8, + R9 = const TrapContext::OFFSET_R9, + R10 = const TrapContext::OFFSET_R10, + R11 = const TrapContext::OFFSET_R11, + R12 = const TrapContext::OFFSET_R12, + R13 = const TrapContext::OFFSET_R13, + R14 = const TrapContext::OFFSET_R14, + R15 = const TrapContext::OFFSET_R15, + RBP = const TrapContext::OFFSET_RBP, + INT_NO = const TrapContext::OFFSET_INT_NO, + CS = const TrapContext::OFFSET_CS, + options(att_syntax), + ) +} + +#[unsafe(naked)] +unsafe extern "C" fn captured_trap_return(trap_context: &mut TrapContext) { + naked_asm!( + ".cfi_startproc", + ".cfi_signal_frame", + ".cfi_def_cfa %rsp, 0x08", + "", + "pushf", + ".cfi_def_cfa_offset 0x10", + "", + "mov %rdi, %rax", + ".cfi_def_cfa %rax, {CS}", + ".cfi_rel_offset %rcx, {RCX}", + ".cfi_rel_offset %rdx, {RDX}", + ".cfi_rel_offset %rdi, {RDI}", + ".cfi_rel_offset %rsi, {RSI}", + ".cfi_rel_offset %r8, {R8}", + ".cfi_rel_offset %r9, {R9}", + ".cfi_rel_offset %r10, {R10}", + ".cfi_rel_offset %r11, {R11}", + ".cfi_rel_offset %rax, {RAX}", + ".cfi_rel_offset %rbx, {RBX}", + ".cfi_rel_offset %r12, {R12}", + ".cfi_rel_offset %r13, {R13}", + ".cfi_rel_offset %r14, {R14}", + ".cfi_rel_offset %r15, {R15}", + ".cfi_rel_offset %rbp, {RBP}", + ".cfi_rel_offset %rflags, {FLAGS}", + ".cfi_rel_offset %rsp, {RSP}", + "", + "mov {RCX}(%rax), %rcx", + ".cfi_restore %rcx", + "mov {RDX}(%rax), %rdx", + ".cfi_restore %rdx", + "mov {RDI}(%rax), %rdi", + ".cfi_restore %rdi", + "mov {RSI}(%rax), %rsi", + ".cfi_restore %rsi", + "mov {R8}(%rax), %r8", + ".cfi_restore %r8", + "mov {R9}(%rax), %r9", + ".cfi_restore %r9", + "mov {R10}(%rax), %r10", + ".cfi_restore %r10", + "mov {R11}(%rax), %r11", + ".cfi_restore %r11", + "", + "xchg %rax, %rsp", + "xchg %rax, {RAX}(%rsp)", + ".cfi_restore %rax", + "xchg %rbx, {RBX}(%rsp)", + ".cfi_restore %rbx", + "xchg %r12, {R12}(%rsp)", + ".cfi_restore %r12", + "xchg %r13, {R13}(%rsp)", + ".cfi_restore %r13", + "xchg %r14, {R14}(%rsp)", + ".cfi_restore %r14", + "xchg %r15, {R15}(%rsp)", + ".cfi_restore %r15", + "xchg %rbp, {RBP}(%rsp)", + ".cfi_restore %rbp", + "", + "cmpq $0x08, {CS}(%rsp)", + "je 2f", + "swapgs", + "", + "2:", + "lea {RIP}(%rsp), %rsp", + ".cfi_def_cfa %rsp, 0x08", + "iretq", + ".cfi_endproc", + RAX = const TrapContext::OFFSET_RAX, + RBX = const TrapContext::OFFSET_RBX, + RCX = const TrapContext::OFFSET_RCX, + RDX = const TrapContext::OFFSET_RDX, + RDI = const TrapContext::OFFSET_RDI, + RSI = const TrapContext::OFFSET_RSI, + R8 = const TrapContext::OFFSET_R8, + R9 = const TrapContext::OFFSET_R9, + R10 = const TrapContext::OFFSET_R10, + R11 = const TrapContext::OFFSET_R11, + R12 = const TrapContext::OFFSET_R12, + R13 = const TrapContext::OFFSET_R13, + R14 = const TrapContext::OFFSET_R14, + R15 = const TrapContext::OFFSET_R15, + RBP = const TrapContext::OFFSET_RBP, + RIP = const TrapContext::OFFSET_RIP, + CS = const TrapContext::OFFSET_CS, + FLAGS = const TrapContext::OFFSET_FLAGS, + RSP = const TrapContext::OFFSET_RSP, + options(att_syntax), + ); +} + +unsafe fn swap_percpu_capturer(new_capturer: usize) -> usize { + let old_capturer: usize; + asm!( + "mov %gs:0x08, {old}", + "mov {new}, %gs:0x08", + new = in(reg) new_capturer, + old = out(reg) old_capturer, + options(att_syntax), + ); + + old_capturer +} + +impl TrapReturn for TrapContext { + unsafe fn trap_return(&mut self) { + let irq_states = disable_irqs_save(); + let old_handler = swap_percpu_capturer(self as *mut _ as usize); + + unsafe { + CPU::local() + .as_mut() + .load_interrupt_stack(self as *mut _ as usize as u64); + } + + captured_trap_return(self); + + swap_percpu_capturer(old_handler); + irq_states.restore(); + } + + unsafe fn trap_return_noreturn(&mut self) -> ! { + disable_irqs(); + + unsafe { + CPU::local() + .as_mut() + .load_interrupt_stack(self as *mut _ as usize as u64); + } + + captured_trap_return(self); + + unreachable!("trap_return_noreturn should not return"); + } +} + +impl IrqStateTrait for IrqState { + fn restore(self) { + let Self(state) = self; + + unsafe { + asm!( + "push {state}", + "popf", + state = in(reg) state, + options(att_syntax, nomem) + ); + } + } +} + +pub fn enable_irqs() { + unsafe { + asm!("sti", options(att_syntax, nomem, nostack)); + } +} + +pub fn disable_irqs() { + unsafe { + asm!("cli", options(att_syntax, nomem, nostack)); + } +} + +pub fn disable_irqs_save() -> IrqState { + let state: u64; + unsafe { + asm!( + "pushf", + "pop {state}", + "cli", + state = out(reg) state, + options(att_syntax, nomem) + ); + } + + IrqState(state) +} diff --git a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs index 196a4ad3..0792bf67 100644 --- a/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs +++ b/crates/eonix_hal/src/arch/x86_64/trap/trap_context.rs @@ -1,16 +1,15 @@ -use crate::processor::CPU; use core::arch::asm; -use eonix_hal_traits::{ - fault::{Fault, PageFaultErrorCode}, - trap::{RawTrapContext, TrapType}, -}; +use core::mem::offset_of; + +use eonix_hal_traits::fault::{Fault, PageFaultErrorCode}; +use eonix_hal_traits::trap::{RawTrapContext, Stack, TrapType}; use eonix_mm::address::VAddr; -#[derive(Clone, Copy, Default)] +use crate::processor::CPU; + #[repr(C, align(16))] +#[derive(Clone, Copy, Default)] pub struct TrapContext { - rax: u64, - rbx: u64, rcx: u64, rdx: u64, rdi: u64, @@ -19,11 +18,17 @@ pub struct TrapContext { r9: u64, r10: u64, r11: u64, + + /// on `trap_return`: save capturer %rsp, load %rax + /// on `trap_entry`: save %rax, load capturer %rsp + rax: u64, + rbx: u64, r12: u64, r13: u64, r14: u64, r15: u64, rbp: u64, + int_no: u64, errcode: u64, rip: u64, @@ -34,6 +39,29 @@ pub struct TrapContext { } impl TrapContext { + pub const OFFSET_RAX: usize = offset_of!(TrapContext, rax); + pub const OFFSET_RBX: usize = offset_of!(TrapContext, rbx); + pub const OFFSET_RCX: usize = offset_of!(TrapContext, rcx); + pub const OFFSET_RDX: usize = offset_of!(TrapContext, rdx); + pub const OFFSET_RDI: usize = offset_of!(TrapContext, rdi); + pub const OFFSET_RSI: usize = offset_of!(TrapContext, rsi); + pub const OFFSET_R8: usize = offset_of!(TrapContext, r8); + pub const OFFSET_R9: usize = offset_of!(TrapContext, r9); + pub const OFFSET_R10: usize = offset_of!(TrapContext, r10); + pub const OFFSET_R11: usize = offset_of!(TrapContext, r11); + pub const OFFSET_R12: usize = offset_of!(TrapContext, r12); + pub const OFFSET_R13: usize = offset_of!(TrapContext, r13); + pub const OFFSET_R14: usize = offset_of!(TrapContext, r14); + pub const OFFSET_R15: usize = offset_of!(TrapContext, r15); + pub const OFFSET_RBP: usize = offset_of!(TrapContext, rbp); + pub const OFFSET_INT_NO: usize = offset_of!(TrapContext, int_no); + pub const OFFSET_ERRCODE: usize = offset_of!(TrapContext, errcode); + pub const OFFSET_RIP: usize = offset_of!(TrapContext, rip); + pub const OFFSET_CS: usize = offset_of!(TrapContext, cs); + pub const OFFSET_FLAGS: usize = offset_of!(TrapContext, flags); + pub const OFFSET_RSP: usize = offset_of!(TrapContext, rsp); + pub const OFFSET_SS: usize = offset_of!(TrapContext, ss); + fn get_fault_type(&self) -> Fault { match self.int_no { 6 | 8 => Fault::InvalidOp, @@ -81,7 +109,7 @@ impl RawTrapContext for TrapContext { type FIrq = impl FnOnce(fn(irqno: usize)); type FTimer = fn(handler: fn()); - fn new() -> Self { + fn blank() -> Self { Self { ..Default::default() } @@ -89,6 +117,7 @@ impl RawTrapContext for TrapContext { fn trap_type(&self) -> TrapType<Self::FIrq, Self::FTimer> { match self.int_no { + 3 => TrapType::Breakpoint, 0..0x20 => TrapType::Fault(self.get_fault_type()), 0x40 => TrapType::Timer { callback: |handler| { @@ -169,12 +198,8 @@ impl RawTrapContext for TrapContext { } fn set_user_call_frame<E>( - &mut self, - pc: usize, - sp: Option<usize>, - ra: Option<usize>, - args: &[usize], - write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, + &mut self, pc: usize, sp: Option<usize>, ra: Option<usize>, + args: &[usize], write_memory: impl Fn(VAddr, &[u8]) -> Result<(), E>, ) -> Result<(), E> { self.set_program_counter(pc); @@ -196,4 +221,30 @@ impl RawTrapContext for TrapContext { self.set_stack_pointer(sp); Ok(()) } + + fn set_kernel_call_frame( + &mut self, pc: usize, stack: &impl Stack, ra: Option<usize>, + args: &[usize], + ) { + let sp = stack.get_bottom().wrapping_sub(1); + + self.set_program_counter(pc); + self.set_stack_pointer(sp.addr()); + + unsafe { + sp.write(ra.map(|f| f as usize).unwrap_or(0)); + } + + if let Some(&arg) = args.get(0) { + self.rdi = arg as _; + } + + if let Some(&arg) = args.get(1) { + self.rsi = arg as _; + } + + if args.len() > 2 { + unimplemented!("More than 2 arguments are not supported for now"); + } + } } diff --git a/crates/eonix_hal/src/context.rs b/crates/eonix_hal/src/context.rs deleted file mode 100644 index c7c1e14a..00000000 --- a/crates/eonix_hal/src/context.rs +++ /dev/null @@ -1 +0,0 @@ -pub use crate::arch::context::TaskContext; diff --git a/crates/eonix_hal/src/lib.rs b/crates/eonix_hal/src/lib.rs index e789ecbb..3d34137e 100644 --- a/crates/eonix_hal/src/lib.rs +++ b/crates/eonix_hal/src/lib.rs @@ -6,12 +6,13 @@ pub(crate) mod arch; pub mod bootstrap; -pub mod context; pub mod mm; pub mod trap; pub mod fence { - pub use crate::arch::fence::{memory_barrier, read_memory_barrier, write_memory_barrier}; + pub use crate::arch::fence::{ + memory_barrier, read_memory_barrier, write_memory_barrier, + }; } pub mod fpu { @@ -19,7 +20,7 @@ pub mod fpu { } pub mod processor { - pub use crate::arch::cpu::{halt, UserTLS, CPU, CPU_COUNT}; + pub use crate::arch::cpu::{halt, CPU, CPU_COUNT}; } /// Re-export the arch module for use in other crates @@ -43,3 +44,43 @@ pub mod arch_exported { pub use eonix_hal_macros::{ap_main, default_trap_handler, main}; pub use eonix_hal_traits as traits; + +#[macro_export] +macro_rules! symbol_addr { + ($sym:expr) => {{ + ($sym) as *const () as usize + }}; + ($sym:expr, $type:ty) => {{ + ($sym) as *const () as *const $type + }}; +} + +#[macro_export] +macro_rules! extern_symbol_addr { + ($sym:ident) => {{ + unsafe extern "C" { + fn $sym(); + } + $crate::symbol_addr!($sym) + }}; + ($sym:ident, $type:ty) => {{ + unsafe extern "C" { + fn $sym(); + } + $crate::symbol_addr!($sym, $type) + }}; +} + +#[macro_export] +macro_rules! extern_symbol_value { + ($sym:ident) => {{ + unsafe extern "C" { + fn $sym(); + } + + static SYMBOL_ADDR: &'static unsafe extern "C" fn() = + &($sym as unsafe extern "C" fn()); + + unsafe { (SYMBOL_ADDR as *const _ as *const usize).read_volatile() } + }}; +} diff --git a/crates/eonix_hal/src/link.x.in b/crates/eonix_hal/src/link.x.in index 81c269c2..eaabdfda 100644 --- a/crates/eonix_hal/src/link.x.in +++ b/crates/eonix_hal/src/link.x.in @@ -18,7 +18,7 @@ SECTIONS { __srodata = .; *(.rodata .rodata.*); - + . = ALIGN(8); PROVIDE(__eh_frame = .); @@ -41,7 +41,7 @@ SECTIONS { } > REGION_DATA AT> LINK_REGION_DATA - .data.after : + .data.after : ALIGN(0x1000) { __data_after = .; } > REGION_DATA AT> LINK_REGION_DATA diff --git a/crates/eonix_hal/src/mm.rs b/crates/eonix_hal/src/mm.rs index 0a5597ac..ef006cb5 100644 --- a/crates/eonix_hal/src/mm.rs +++ b/crates/eonix_hal/src/mm.rs @@ -1,16 +1,14 @@ -use core::{ - alloc::{AllocError, Allocator, Layout}, - cell::RefCell, - ptr::NonNull, -}; -use eonix_mm::{ - address::{AddrOps as _, PRange}, - paging::{PageAlloc, UnmanagedRawPage, PAGE_SIZE, PFN}, -}; +use core::alloc::{AllocError, Allocator, Layout}; +use core::cell::RefCell; +use core::ptr::NonNull; + +use eonix_mm::address::{AddrOps as _, PRange}; +use eonix_mm::page_table::PageTableAlloc; +use eonix_mm::paging::{BasicFolio, FrameAlloc, PAGE_SIZE, PFN}; pub use crate::arch::mm::{ - flush_tlb, flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchMemory, - ArchPagingMode, ArchPhysAccess, GLOBAL_PAGE_TABLE, + flush_tlb, flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, + ArchMemory, ArchPhysAccess, GLOBAL_PAGE_TABLE, }; pub struct BasicPageAlloc { @@ -43,9 +41,8 @@ impl BasicPageAlloc { fn alloc_one(&mut self) -> PFN { assert_ne!(self.head, self.tail, "No free pages available"); let mut range = self.ranges[self.head].take().unwrap(); - range = range.shrink(PAGE_SIZE); - - let pfn = PFN::from(range.end()); + let pfn = PFN::from(range.start()); + range = PRange::new(range.start() + PAGE_SIZE, range.end()); if range.len() != 0 { self.ranges[self.head] = Some(range); @@ -89,7 +86,8 @@ impl BasicPageAlloc { panic!("Page allocator is full"); } - self.ranges[tail] = Some(PRange::new(range.start().ceil(), range.end().floor())); + self.ranges[tail] = + Some(PRange::new(range.start().ceil(), range.end().floor())); } pub fn alloc(&mut self, order: u32) -> PFN { @@ -118,19 +116,23 @@ impl<'a> BasicPageAllocRef<'a> { } } -impl PageAlloc for BasicPageAllocRef<'_> { - type RawPage = UnmanagedRawPage; +impl FrameAlloc for BasicPageAllocRef<'_> { + type Folio = BasicFolio; - fn alloc_order(&self, order: u32) -> Option<Self::RawPage> { - Some(Self::RawPage::new(self.0.borrow_mut().alloc(order), order)) + fn alloc_order(&self, order: u32) -> Option<Self::Folio> { + Some(BasicFolio::new(self.0.borrow_mut().alloc(order), order)) } +} + +impl PageTableAlloc for BasicPageAllocRef<'_> { + type Folio = BasicFolio; - unsafe fn dealloc(&self, _: Self::RawPage) { - panic!("Dealloc is not supported in BasicPageAlloc"); + fn alloc(&self) -> Self::Folio { + FrameAlloc::alloc(self).unwrap() } - fn has_management_over(&self, _: Self::RawPage) -> bool { - true + unsafe fn from_raw(&self, pfn: PFN) -> Self::Folio { + BasicFolio::new(pfn, 0) } } @@ -145,7 +147,10 @@ impl<'a> ScopedAllocator<'a> { } } - pub fn with_alloc<'b, 'r, O>(&'r self, func: impl FnOnce(&'b ScopedAllocator<'a>) -> O) -> O + pub fn with_alloc<'b, 'r, O>( + &'r self, + func: impl FnOnce(&'b ScopedAllocator<'a>) -> O, + ) -> O where 'a: 'b, 'r: 'b, diff --git a/crates/eonix_log/src/lib.rs b/crates/eonix_log/src/lib.rs index 92b1639f..01b6a587 100644 --- a/crates/eonix_log/src/lib.rs +++ b/crates/eonix_log/src/lib.rs @@ -2,6 +2,7 @@ use alloc::sync::Arc; use core::fmt::{self, Write}; + use eonix_sync::{Spin, SpinIrq as _}; extern crate alloc; @@ -91,18 +92,31 @@ macro_rules! println_fatal { #[macro_export] macro_rules! println_trace { - ($feat:literal) => { + (feat:$feat:literal) => { #[deny(unexpected_cfgs)] { #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] ") + $crate::println!("[kernel:trace]") } }; - ($feat:literal, $($arg:tt)*) => {{ + (feat:$feat:literal, $fmt:literal) => {{ #[deny(unexpected_cfgs)] { #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] {}", format_args!($($arg)*)) + $crate::println!(concat!("[kernel:trace] ", $feat)) } }}; + (feat:$feat:literal, $fmt:literal, $($arg:expr $(,)?)*) => { + #[deny(unexpected_cfgs)] + { + // Suppress unused variables warning + #[cfg(not(feature = $feat))] + { + $(let _ = $arg;)* + } + + #[cfg(feature = $feat)] + $crate::println!("[kernel:trace] {}", format_args!($fmt, $($arg,)*)) + } + }; } diff --git a/crates/eonix_mm/src/address/paddr.rs b/crates/eonix_mm/src/address/paddr.rs index 6fadbd2a..bbfa299e 100644 --- a/crates/eonix_mm/src/address/paddr.rs +++ b/crates/eonix_mm/src/address/paddr.rs @@ -1,11 +1,11 @@ +use core::fmt; +use core::ops::{Add, Sub}; +use core::ptr::NonNull; + use super::addr::Addr; use crate::paging::{PAGE_SIZE_BITS, PFN}; -use core::{ - fmt, - ops::{Add, Sub}, - ptr::NonNull, -}; +/// Convert PAddr to VAddr. pub trait PhysAccess { /// Translate the data that this address is pointing to into kernel /// accessible pointer. Use it with care. diff --git a/crates/eonix_mm/src/page_table.rs b/crates/eonix_mm/src/page_table.rs index 55732f72..f3528060 100644 --- a/crates/eonix_mm/src/page_table.rs +++ b/crates/eonix_mm/src/page_table.rs @@ -3,7 +3,7 @@ mod paging_mode; mod pte; mod pte_iterator; -pub use page_table::{PageTable, RawPageTable}; +pub use page_table::{PageTable, PageTableAlloc, RawPageTable}; pub use paging_mode::{PageTableLevel, PagingMode}; pub use pte::{PageAttribute, RawAttribute, TableAttribute, PTE}; pub use pte_iterator::PageTableIterator; diff --git a/crates/eonix_mm/src/page_table/page_table.rs b/crates/eonix_mm/src/page_table/page_table.rs index 8318049f..80be63b9 100644 --- a/crates/eonix_mm/src/page_table/page_table.rs +++ b/crates/eonix_mm/src/page_table/page_table.rs @@ -1,15 +1,12 @@ -use super::{ - paging_mode::PageTableLevel, - pte::{RawAttribute, TableAttribute}, - pte_iterator::{KernelIterator, UserIterator}, - PagingMode, PTE, -}; -use crate::{ - address::{PAddr, VRange}, - page_table::PageTableIterator, - paging::{GlobalPageAlloc, Page, PageAccess, PageAlloc, PageBlock}, -}; -use core::{marker::PhantomData, ptr::NonNull}; +use core::marker::PhantomData; +use core::ptr::NonNull; + +use super::paging_mode::PageTableLevel; +use super::pte::{RawAttribute, TableAttribute}; +use super::{PagingMode, PTE}; +use crate::address::{PAddr, VRange}; +use crate::page_table::PageTableIterator; +use crate::paging::{Folio, PageAccess, PageBlock, PFN}; pub trait RawPageTable<'a>: Send + 'a { type Entry: PTE + 'a; @@ -24,45 +21,60 @@ pub trait RawPageTable<'a>: Send + 'a { unsafe fn from_ptr(ptr: NonNull<PageBlock>) -> Self; } +pub trait PageTableAlloc: Clone { + type Folio: Folio; + + fn alloc(&self) -> Self::Folio; + unsafe fn from_raw(&self, pfn: PFN) -> Self::Folio; +} + +pub trait GlobalPageTableAlloc: PageTableAlloc { + const GLOBAL: Self; +} + pub struct PageTable<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { - root_table_page: Page<A>, - phantom: PhantomData<&'a (M, X)>, + root_table_page: A::Folio, + alloc: A, + access: X, + phantom: PhantomData<&'a M>, } impl<'a, M, A, X> PageTable<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { - pub fn with_root_table(root_table_page: Page<A>) -> Self { + pub fn new(root_table_page: A::Folio, alloc: A, access: X) -> Self { Self { root_table_page, + alloc, + access, phantom: PhantomData, } } pub fn clone_global<'b, B>(&self) -> PageTable<'b, M, B, X> where - B: GlobalPageAlloc, + B: GlobalPageTableAlloc, { - self.clone_in(B::global()) + self.clone_in(B::GLOBAL) } pub fn clone_in<'b, B>(&self, alloc: B) -> PageTable<'b, M, B, X> where - B: PageAlloc, + B: PageTableAlloc, { - let new_root_table_page = Page::alloc_in(alloc); - let new_table_data = X::get_ptr_for_page(&new_root_table_page); - let kernel_table_data = X::get_ptr_for_page(&self.root_table_page); + let new_root_table_page = alloc.alloc(); + let new_table_data = self.access.get_ptr_for_page(&new_root_table_page); + let kernel_table_data = self.access.get_ptr_for_page(&self.root_table_page); unsafe { // SAFETY: `new_table_data` and `kernel_table_data` are both valid pointers @@ -82,7 +94,7 @@ where root_page_table.index_mut(idx).take(); } - PageTable::with_root_table(new_root_table_page) + PageTable::new(new_root_table_page, alloc, self.access.clone()) } pub fn addr(&self) -> PAddr { @@ -90,100 +102,59 @@ where } pub fn iter_user(&self, range: VRange) -> impl Iterator<Item = &mut M::Entry> { - let alloc = self.root_table_page.allocator(); - let page_table_ptr = X::get_ptr_for_page(&self.root_table_page); + let page_table_ptr = self.access.get_ptr_for_page(&self.root_table_page); let root_page_table = unsafe { // SAFETY: `page_table_ptr` is a valid pointer to a page table. M::RawTable::from_ptr(page_table_ptr) }; - PageTableIterator::<M, A, X, UserIterator>::new(root_page_table, range, alloc.clone()) - } - - pub fn iter_kernel(&self, range: VRange) -> impl Iterator<Item = &mut M::Entry> { - self.iter_kernel_levels(range, M::LEVELS) + PageTableIterator::<M, _, _>::new( + root_page_table, + range, + TableAttribute::USER, + self.alloc.clone(), + self.access.clone(), + ) } - /// Iterates over the kernel space entries in the page table for the specified levels. - /// - /// # Parameters - /// - `range`: The virtual address range to iterate over. - /// - `levels`: A slice of `PageTableLevel` that specifies which levels of the page table - /// should be included in the iteration. Each level corresponds to a level in the page - /// table hierarchy, and the iterator will traverse entries at these levels. + /// Iterates over the kernel space entries in the page table. /// /// # Returns /// An iterator over mutable references to the page table entries (`M::Entry`) within the - /// specified range and levels. + /// specified range. /// /// # Example /// ``` /// let range = VRange::new(0x1234000, 0x1300000); - /// let levels = &M::LEVELS[..2]; - /// for pte in page_table.iter_kernel_levels(range, levels) { + /// for pte in page_table.iter_kernel(range) { /// // Process each entry /// } /// ``` - pub fn iter_kernel_levels( - &self, - range: VRange, - levels: &'static [PageTableLevel], - ) -> impl Iterator<Item = &mut M::Entry> { - self.iter_kernel_in(range, levels, self.root_table_page.allocator()) - } - - /// Iterates over the kernel space entries in the page table for the specified levels - /// with a given page allocator. - /// - /// # Parameters - /// - `range`: The virtual address range to iterate over. - /// - `levels`: A slice of `PageTableLevel` that specifies which levels of the page table - /// should be included in the iteration. Each level corresponds to a level in the page - /// table hierarchy, and the iterator will traverse entries at these levels. - /// - `alloc`: A page allocator that provides memory for the page table entries. - /// - /// # Returns - /// An iterator over mutable references to the page table entries (`M::Entry`) within the - /// specified range and levels. - /// - /// # Example - /// ```no_run - /// let range = VRange::new(0x1234000, 0x1300000); - /// let levels = &M::LEVELS[..2]; - /// for pte in page_table.iter_kernel_in(range, levels, NoAlloc) { - /// // Process each entry - /// } - /// ``` - pub fn iter_kernel_in<A1: PageAlloc>( - &self, - range: VRange, - levels: &'static [PageTableLevel], - alloc: A1, - ) -> impl Iterator<Item = &mut M::Entry> { - let page_table_ptr = X::get_ptr_for_page(&self.root_table_page); + pub fn iter_kernel(&self, range: VRange) -> impl Iterator<Item = &mut M::Entry> { + let page_table_ptr = self.access.get_ptr_for_page(&self.root_table_page); let root_page_table = unsafe { // SAFETY: `page_table_ptr` is a valid pointer to a page table. M::RawTable::from_ptr(page_table_ptr) }; - PageTableIterator::<M, A1, X, KernelIterator>::with_levels( + PageTableIterator::<M, _, _>::with_levels( root_page_table, range, - alloc, - levels, + TableAttribute::GLOBAL, + self.alloc.clone(), + self.access.clone(), + M::LEVELS, ) } - fn drop_page_table_recursive(page_table: &Page<A>, levels: &[PageTableLevel]) { + fn drop_page_table_recursive(&self, page_table: &A::Folio, levels: &[PageTableLevel]) { let [level, remaining_levels @ ..] = levels else { return }; if remaining_levels.is_empty() { // We reached the last level, no need to go deeper. return; } - let alloc = page_table.allocator(); - - let page_table_ptr = X::get_ptr_for_page(page_table); + let page_table_ptr = self.access.get_ptr_for_page(page_table); let mut page_table = unsafe { // SAFETY: `page_table_ptr` is a valid pointer to a page table. M::RawTable::from_ptr(page_table_ptr) @@ -201,10 +172,10 @@ where let page_table = unsafe { // SAFETY: We got the pfn from a valid page table entry, so it should be valid. - Page::from_raw_in(pfn, alloc.clone()) + self.alloc.from_raw(pfn) }; - Self::drop_page_table_recursive(&page_table, remaining_levels); + self.drop_page_table_recursive(&page_table, remaining_levels); } } } @@ -213,10 +184,10 @@ impl<'a, M, A, X> Drop for PageTable<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { fn drop(&mut self) { - Self::drop_page_table_recursive(&self.root_table_page, M::LEVELS); + self.drop_page_table_recursive(&self.root_table_page, M::LEVELS); } } diff --git a/crates/eonix_mm/src/page_table/pte.rs b/crates/eonix_mm/src/page_table/pte.rs index e067d207..c14d5738 100644 --- a/crates/eonix_mm/src/page_table/pte.rs +++ b/crates/eonix_mm/src/page_table/pte.rs @@ -1,6 +1,7 @@ -use crate::paging::PFN; use bitflags::bitflags; +use crate::paging::PFN; + bitflags! { #[derive(Clone, Copy, PartialEq)] pub struct TableAttribute: usize { diff --git a/crates/eonix_mm/src/page_table/pte_iterator.rs b/crates/eonix_mm/src/page_table/pte_iterator.rs index 89b9fb9f..a9e4ff46 100644 --- a/crates/eonix_mm/src/page_table/pte_iterator.rs +++ b/crates/eonix_mm/src/page_table/pte_iterator.rs @@ -1,62 +1,14 @@ -use super::{ - pte::{RawAttribute, TableAttribute}, - PageTableLevel, PagingMode, RawPageTable as _, PTE, -}; -use crate::{ - address::{AddrOps as _, VRange}, - paging::{Page, PageAccess, PageAlloc}, -}; -use core::{marker::PhantomData}; - -pub struct KernelIterator; -pub struct UserIterator; - -pub trait IteratorType<M: PagingMode> { - fn page_table_attributes() -> TableAttribute; - - fn get_page_table<'a, A, X>(pte: &mut M::Entry, alloc: &A) -> M::RawTable<'a> - where - A: PageAlloc, - X: PageAccess, - { - let attr = pte.get_attr().as_table_attr().expect("Not a page table"); - - if attr.contains(TableAttribute::PRESENT) { - let pfn = pte.get_pfn(); - unsafe { - // SAFETY: We are creating a pointer to a page referenced to in - // some page table, which should be valid. - let page_table_ptr = X::get_ptr_for_pfn(pfn); - // SAFETY: `page_table_ptr` is a valid pointer to a page table. - M::RawTable::from_ptr(page_table_ptr) - } - } else { - let page = Page::alloc_in(alloc.clone()); - let page_table_ptr = X::get_ptr_for_page(&page); - - unsafe { - // SAFETY: `page_table_ptr` is good for writing and properly aligned. - page_table_ptr.write_bytes(0, 1); - } - - pte.set( - page.into_raw(), - <M::Entry as PTE>::Attr::from(Self::page_table_attributes()), - ); - - unsafe { - // SAFETY: `page_table_ptr` is a valid pointer to a page table. - M::RawTable::from_ptr(page_table_ptr) - } - } - } -} +use super::page_table::PageTableAlloc; +use super::pte::{RawAttribute, TableAttribute}; +use super::{PageTableLevel, PagingMode, RawPageTable as _, PTE}; +use crate::address::{AddrOps as _, VRange}; +use crate::paging::{Folio, PageAccess}; -pub struct PageTableIterator<'a, M, A, X, K> +pub struct PageTableIterator<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, { /// Specifies the hierarchy of page table levels to iterate over. @@ -69,19 +21,19 @@ where indicies: [u16; 8], tables: [Option<M::RawTable<'a>>; 8], + fill_entry_attr: TableAttribute, + alloc: A, - _phantom: PhantomData<&'a (X, K)>, + access: X, } -impl<'a, M, A, X, K> PageTableIterator<'a, M, A, X, K> +impl<'a, M, A, X> PageTableIterator<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, - K: IteratorType<M>, { - fn parse_tables_starting_from(&mut self, idx_level: usize) { for (idx, &pt_idx) in self .indicies @@ -98,18 +50,58 @@ where }; let parent_table = parent_table.as_mut().expect("Parent table is None"); let next_pte = parent_table.index_mut(pt_idx); - child_table.replace(K::get_page_table::<A, X>(next_pte, &self.alloc)); + + child_table.replace({ + let attr = next_pte + .get_attr() + .as_table_attr() + .expect("Not a page table"); + + if attr.contains(TableAttribute::PRESENT) { + let pfn = next_pte.get_pfn(); + unsafe { + // SAFETY: We are creating a pointer to a page referenced to in + // some page table, which should be valid. + let page_table_ptr = self.access.get_ptr_for_pfn(pfn); + // SAFETY: `page_table_ptr` is a valid pointer to a page table. + M::RawTable::from_ptr(page_table_ptr) + } + } else { + let page = self.alloc.alloc(); + let page_table_ptr = self.access.get_ptr_for_page(&page); + + unsafe { + // SAFETY: `page_table_ptr` is good for writing and properly aligned. + page_table_ptr.write_bytes(0, 1); + } + + next_pte.set(page.into_raw(), self.fill_entry_attr.into()); + + unsafe { + // SAFETY: `page_table_ptr` is a valid pointer to a page table. + M::RawTable::from_ptr(page_table_ptr) + } + } + }); } } - pub fn new(page_table: M::RawTable<'a>, range: VRange, alloc: A) -> Self { - Self::with_levels(page_table, range, alloc, M::LEVELS) + pub fn new( + page_table: M::RawTable<'a>, + range: VRange, + fill_entry_attr: TableAttribute, + alloc: A, + access: X, + ) -> Self { + Self::with_levels(page_table, range, fill_entry_attr, alloc, access, M::LEVELS) } pub fn with_levels( page_table: M::RawTable<'a>, range: VRange, + fill_entry_attr: TableAttribute, alloc: A, + access: X, levels: &'static [PageTableLevel], ) -> Self { let start = range.start().floor(); @@ -122,8 +114,9 @@ where remaining: (end - start) / last_level.page_size(), indicies: [0; 8], tables: [const { None }; 8], + fill_entry_attr: fill_entry_attr.union(TableAttribute::PRESENT), alloc, - _phantom: PhantomData, + access, }; for (i, level) in levels.iter().enumerate() { @@ -137,13 +130,12 @@ where } } -impl<'a, M, A, X, K> Iterator for PageTableIterator<'a, M, A, X, K> +impl<'a, M, A, X> Iterator for PageTableIterator<'a, M, A, X> where M: PagingMode, M::Entry: 'a, - A: PageAlloc, + A: PageTableAlloc, X: PageAccess, - K: IteratorType<M>, { type Item = &'a mut M::Entry; @@ -178,15 +170,3 @@ where Some(retval) } } - -impl<M: PagingMode> IteratorType<M> for KernelIterator { - fn page_table_attributes() -> TableAttribute { - TableAttribute::PRESENT | TableAttribute::GLOBAL - } -} - -impl<M: PagingMode> IteratorType<M> for UserIterator { - fn page_table_attributes() -> TableAttribute { - TableAttribute::PRESENT | TableAttribute::USER - } -} diff --git a/crates/eonix_mm/src/page_table/walk.rs b/crates/eonix_mm/src/page_table/walk.rs new file mode 100644 index 00000000..aba80b09 --- /dev/null +++ b/crates/eonix_mm/src/page_table/walk.rs @@ -0,0 +1,210 @@ +use super::pte::{RawAttribute, TableAttribute}; +use super::{PageTableLevel, PTE}; +use crate::address::{AddrOps, VAddr, VRange}; +use crate::paging::PFN; + +pub enum WalkState { + Next, + Skip, + Break, +} + +pub trait PageTable: Sized { + type Entry: PTE; + const LEVELS: &'static [PageTableLevel]; + + fn index(&self, index: usize) -> &Self::Entry; + fn index_mut(&mut self, index: usize) -> &mut Self::Entry; + + fn from_pfn(pfn: PFN) -> Self; + unsafe fn take_pfn(pfn: PFN) -> Self; +} + +pub struct PageTableWalk<'a, T, D> +where + T: PageTable, +{ + levels: &'a [PageTableLevel], + fill_entry: &'a [fn(&mut D, &mut T::Entry) -> Option<PFN>], + walk_entry: &'a [fn(&mut D, &mut T::Entry) -> WalkState], + data: D, +} + +fn try_get_table<T, D>( + entry: &mut T::Entry, + data: &mut D, + fill_entry: fn(&mut D, &mut T::Entry) -> Option<PFN>, +) -> Option<T> +where + T: PageTable, +{ + let (mut pfn, attr) = entry.get(); + + // Always skip huge page entries + let attr = attr.as_table_attr()?; + + // For normal entries, check present flags + if !attr.contains(TableAttribute::PRESENT) { + // Skip entries filled with nothing + pfn = fill_entry(data, entry)?; + } + + Some(T::from_pfn(pfn)) +} + +fn _walk_page_table<T, D>( + walk: &mut PageTableWalk<T, D>, + cur_level: usize, + table: &mut T, + range: VRange, +) where + T: PageTable, +{ + let level = walk.levels[cur_level]; + + let page_size = level.page_size(); + let mut addr = range.start(); + + while addr < range.end() { + let idx = level.index_of(addr); + let entry = table.index_mut(idx); + + let mut next_table = None; + if cur_level < walk.levels.len() - 1 { + next_table = try_get_table(entry, &mut walk.data, walk.fill_entry[cur_level]); + } + + match ( + walk.walk_entry[cur_level](&mut walk.data, entry), + &mut next_table, + ) { + (WalkState::Break, _) => break, + (WalkState::Next, Some(next_table)) => _walk_page_table( + walk, + cur_level + 1, + next_table, + VRange::new(addr, range.end()), + ), + // `fill_entry` says that we shouldn't continue. + (WalkState::Next, None) => {} + _ => {} + } + + addr = addr.floor_to(page_size) + page_size; + } +} + +pub fn walk_page_table<T, D>(walk: &mut PageTableWalk<T, D>, table: &mut T, range: VRange) +where + T: PageTable, +{ + _walk_page_table(walk, 0, table, range); +} + +pub fn drop_user_page_table<T>(mut root_page_table: T) +where + T: PageTable, +{ + fn walk<T: PageTable, const LEVEL: usize>(_: &mut (), entry: &mut T::Entry) -> WalkState { + let (pfn, attr) = entry.get(); + let Some(attr) = attr.as_table_attr() else { + return WalkState::Skip; + }; + + if !attr.contains(TableAttribute::USER) { + return WalkState::Skip; + } + + unsafe { + // Check `_walk_page_table`: We will and only will touch the next level of table with + // `next_table` holding a refcount. We take the table away from the parent table now. + T::take_pfn(pfn); + } + + entry.set(PFN::from_val(0), TableAttribute::empty().into()); + + if LEVEL == 2 { + WalkState::Skip + } else { + WalkState::Next + } + } + + let mut walk = PageTableWalk { + levels: T::LEVELS, + fill_entry: &[no_fill::<T, ()>, no_fill::<T, ()>, no_fill::<T, ()>], + walk_entry: &[walk::<T, 0>, walk::<T, 1>, walk::<T, 2>, skip_walk::<T, ()>], + data: (), + }; + + walk_page_table( + &mut walk, + &mut root_page_table, + VRange::new(VAddr::from(0), VAddr::from(0x0000_8000_0000_0000)), + ); +} + +pub fn iter_pte<T: PageTable>( + page_table: &mut T, + range: VRange, + fill_func: impl FnMut(&mut T::Entry) -> Option<PFN>, + for_each: impl FnMut(&mut T::Entry), +) { + let walker = (fill_func, for_each); + + fn fill_entry<T: PageTable>( + (fill, _): &mut ( + impl FnMut(&mut T::Entry) -> Option<PFN>, + impl FnMut(&mut T::Entry), + ), + entry: &mut T::Entry, + ) -> Option<PFN> { + fill(entry) + } + + fn walk_entry<T: PageTable>( + (_, for_each): &mut ( + impl FnMut(&mut T::Entry) -> Option<PFN>, + impl FnMut(&mut T::Entry), + ), + entry: &mut T::Entry, + ) -> WalkState { + for_each(entry); + WalkState::Next + } + + let mut walk = PageTableWalk { + levels: T::LEVELS, + fill_entry: &[fill_entry::<T>, fill_entry::<T>, fill_entry::<T>], + walk_entry: &[ + cont_walk::<T, _>, + cont_walk::<T, _>, + cont_walk::<T, _>, + walk_entry::<T>, + ], + data: walker, + }; + + walk_page_table(&mut walk, page_table, range); +} + +pub fn no_fill<T, D>(_: &mut D, _: &mut T::Entry) -> Option<PFN> +where + T: PageTable, +{ + None +} + +pub fn skip_walk<T, D>(_: &mut D, _: &mut T::Entry) -> WalkState +where + T: PageTable, +{ + WalkState::Skip +} + +pub fn cont_walk<T, D>(_: &mut D, _: &mut T::Entry) -> WalkState +where + T: PageTable, +{ + WalkState::Next +} diff --git a/crates/eonix_mm/src/paging.rs b/crates/eonix_mm/src/paging.rs index 88da902e..f0166cf3 100644 --- a/crates/eonix_mm/src/paging.rs +++ b/crates/eonix_mm/src/paging.rs @@ -1,9 +1,11 @@ +mod list; mod page; mod page_alloc; mod pfn; -mod raw_page; +mod zone; -pub use page::{Page, PageAccess, PageBlock, PAGE_SIZE, PAGE_SIZE_BITS}; -pub use page_alloc::{GlobalPageAlloc, NoAlloc, PageAlloc}; +pub use list::{FolioList, FolioListSized}; +pub use page::{BasicFolio, Folio, PageAccess, PageBlock, PAGE_SIZE, PAGE_SIZE_BITS}; +pub use page_alloc::{FrameAlloc, GlobalFrameAlloc}; pub use pfn::PFN; -pub use raw_page::{RawPage, UnmanagedRawPage}; +pub use zone::Zone; diff --git a/crates/eonix_mm/src/paging/list.rs b/crates/eonix_mm/src/paging/list.rs new file mode 100644 index 00000000..2dd557c9 --- /dev/null +++ b/crates/eonix_mm/src/paging/list.rs @@ -0,0 +1,19 @@ +pub trait FolioList { + type Folio; + + fn is_empty(&self) -> bool; + + fn peek_head(&mut self) -> Option<&mut Self::Folio>; + + fn pop_head(&mut self) -> Option<&'static mut Self::Folio>; + fn push_tail(&mut self, page: &'static mut Self::Folio); + fn remove(&mut self, page: &mut Self::Folio); +} + +pub trait FolioListSized: FolioList + Sized { + const NEW: Self; + + fn new() -> Self { + Self::NEW + } +} diff --git a/crates/eonix_mm/src/paging/page.rs b/crates/eonix_mm/src/paging/page.rs index c5a14b5e..8b067e43 100644 --- a/crates/eonix_mm/src/paging/page.rs +++ b/crates/eonix_mm/src/paging/page.rs @@ -1,6 +1,8 @@ -use super::{GlobalPageAlloc, PageAlloc, RawPage as _, PFN}; -use crate::address::{AddrRange, PAddr, PhysAccess}; -use core::{fmt, mem::ManuallyDrop, ptr::NonNull, sync::atomic::Ordering}; +use core::mem::ManuallyDrop; +use core::ptr::NonNull; + +use super::PFN; +use crate::address::{PAddr, PRange}; pub const PAGE_SIZE: usize = 4096; pub const PAGE_SIZE_BITS: u32 = PAGE_SIZE.trailing_zeros(); @@ -15,306 +17,81 @@ pub struct PageBlock([u8; PAGE_SIZE]); /// A trait that provides the kernel access to the page. #[doc(notable_trait)] -pub trait PageAccess { +pub trait PageAccess: Clone { /// Returns a kernel-accessible pointer to the page referenced by the given /// physical frame number. /// /// # Safety /// This function is unsafe because calling this function on some non-existing /// pfn will cause undefined behavior. - unsafe fn get_ptr_for_pfn(pfn: PFN) -> NonNull<PageBlock>; + unsafe fn get_ptr_for_pfn(&self, pfn: PFN) -> NonNull<PageBlock>; /// Returns a kernel-accessible pointer to the given page. - fn get_ptr_for_page<A: PageAlloc>(page: &Page<A>) -> NonNull<PageBlock> { + fn get_ptr_for_page<F: Folio>(&self, page: &F) -> NonNull<PageBlock> { unsafe { // SAFETY: `page.pfn()` is guaranteed to be valid. - Self::get_ptr_for_pfn(page.pfn()) + self.get_ptr_for_pfn(page.pfn()) } } } -/// A Page allocated in allocator `A`. -#[derive(PartialEq, Eq, PartialOrd, Ord)] -pub struct Page<A: PageAlloc> { - raw_page: A::RawPage, - alloc: A, -} - -unsafe impl<A: PageAlloc> Send for Page<A> {} -unsafe impl<A: PageAlloc> Sync for Page<A> {} - -impl<A> Page<A> -where - A: GlobalPageAlloc, -{ - /// Allocate a page of the given *order*. - pub fn alloc_order(order: u32) -> Self { - Self::alloc_order_in(order, A::global()) - } - - /// Allocate exactly one page. - pub fn alloc() -> Self { - Self::alloc_in(A::global()) - } +/// A [`Folio`] represents one page or a bunch of adjacent pages. +pub trait Folio { + /// Returns the physical frame number of the folio, which is aligned with + /// the folio's size and valid. + fn pfn(&self) -> PFN; - /// Allocate a contiguous block of pages that can contain at least `count` pages. - pub fn alloc_at_least(count: usize) -> Self { - Self::alloc_at_least_in(count, A::global()) - } + /// Returns the folio's *order* (log2 of the number of pages contained in + /// the folio). + fn order(&self) -> u32; - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to a valid page allocated through `alloc_order()` and that the - /// page have not been freed or deallocated yet. - /// - /// No checks are done. Any violation of this assumption may lead to undefined behavior. - pub unsafe fn from_raw_unchecked(pfn: PFN) -> Self { - unsafe { Self::from_raw_unchecked_in(pfn, A::global()) } - } - - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// This function is a safe wrapper around `from_paddr_unchecked()` that does **some sort - /// of** checks to ensure that the page is valid and managed by the allocator. - /// - /// # Panic - /// This function will panic if the page is not valid or if the page is not managed by - /// the allocator. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to an existing page (A.K.A. inside the global page array) and the - /// page will not be freed or deallocated during the call. - pub unsafe fn from_raw(pfn: PFN) -> Self { - unsafe { Self::from_raw_in(pfn, A::global()) } - } - - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_in()`. - /// - /// # Safety - /// Check `from_raw()` for the safety requirements. - pub unsafe fn with_raw<F, O>(pfn: PFN, func: F) -> O - where - F: FnOnce(&Self) -> O, - { - unsafe { Self::with_raw_in(pfn, A::global(), func) } - } - - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_unchecked_in()`. - /// - /// # Safety - /// Check `from_raw_unchecked()` for the safety requirements. - pub unsafe fn with_raw_unchecked<F, O>(pfn: PFN, func: F, alloc: A) -> O - where - F: FnOnce(&Self) -> O, - { - unsafe { Self::with_raw_unchecked_in(pfn, func, alloc) } - } -} - -impl<A> Page<A> -where - A: PageAlloc, -{ - /// Allocate a page of the given *order*. - pub fn alloc_order_in(order: u32, alloc: A) -> Self { - Self { - raw_page: alloc.alloc_order(order).expect("Out of memory"), - alloc, - } - } - - /// Allocate exactly one page. - pub fn alloc_in(alloc: A) -> Self { - Self { - raw_page: alloc.alloc().expect("Out of memory"), - alloc, - } - } - - /// Allocate a contiguous block of pages that can contain at least `count` pages. - pub fn alloc_at_least_in(count: usize, alloc: A) -> Self { - Self { - raw_page: alloc.alloc_at_least(count).expect("Out of memory"), - alloc, - } - } - - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to a valid page managed by `alloc` and that the page have not - /// been freed or deallocated yet. - /// - /// No checks are done. Any violation of this assumption may lead to undefined behavior. - pub unsafe fn from_raw_unchecked_in(pfn: PFN, alloc: A) -> Self { - Self { - raw_page: A::RawPage::from(pfn), - alloc, - } + /// Returns the total size of the folio in bytes. + fn len(&self) -> usize { + 1 << (self.order() + PAGE_SIZE_BITS) } - /// Acquire the ownership of the page pointed to by `pfn`, leaving `refcount` untouched. - /// - /// This function is a safe wrapper around `from_paddr_unchecked()` that does **some sort - /// of** checks to ensure that the page is valid and managed by the allocator. - /// - /// # Panic - /// This function will panic if the page is not valid or if the page is not managed by - /// the allocator. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller has ensured that - /// `pfn` points to an existing page (A.K.A. inside the global page array) and the - /// page will not be freed or deallocated during the call. - pub unsafe fn from_raw_in(pfn: PFN, alloc: A) -> Self { - unsafe { - // SAFETY: The caller guarantees that the page is inside the global page array. - assert!(alloc.has_management_over(A::RawPage::from(pfn))); - - // SAFETY: We've checked that the validity of the page. And the caller guarantees - // that the page will not be freed or deallocated during the call. - Self::from_raw_unchecked_in(pfn, alloc) - } + /// Returns the start physical address of the folio, which is guaranteed to + /// be aligned to the folio's size and valid. + fn start(&self) -> PAddr { + PAddr::from(self.pfn()) } - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_in()`. - /// - /// # Safety - /// Check `from_raw_in()` for the safety requirements. - pub unsafe fn with_raw_in<F, O>(pfn: PFN, alloc: A, func: F) -> O - where - F: FnOnce(&Self) -> O, - { - unsafe { - let me = ManuallyDrop::new(Self::from_raw_in(pfn, alloc)); - func(&me) - } + /// Returns the physical address range of the ifolio, which is guaranteed to + /// be aligned to the folio's size and valid. + fn range(&self) -> PRange { + PRange::from(self.start()).grow(self.len()) } - /// Do some work with the page without touching the reference count with the same - /// restrictions as `from_raw_unchecked_in()`. - /// - /// # Safety - /// Check `from_raw_unchecked_in()` for the safety requirements. - pub unsafe fn with_raw_unchecked_in<F, O>(pfn: PFN, func: F, alloc: A) -> O + /// Consumes the folio and returns the PFN without dropping the reference + /// count the folio holds. + fn into_raw(self) -> PFN where - F: FnOnce(&Self) -> O, + Self: Sized, { - unsafe { - let me = ManuallyDrop::new(Self::from_raw_unchecked_in(pfn, alloc)); - func(&me) - } - } - - /// Whether we are the only owner of the page. - pub fn is_exclusive(&self) -> bool { - self.raw_page.refcount().load(Ordering::Acquire) == 1 - } - - /// Returns the *order* of the page, which is the log2 of the number of pages - /// contained in the page object. - pub fn order(&self) -> u32 { - self.raw_page.order() - } - - /// Returns the total size of the page in bytes. - pub fn len(&self) -> usize { - 1 << (self.order() + PAGE_SIZE_BITS) - } - - /// Consumes the `Page` and returns the physical frame number without dropping - /// the reference count the page holds. - pub fn into_raw(self) -> PFN { let me = ManuallyDrop::new(self); me.pfn() } - - /// Returns the physical frame number of the page, which is aligned with the - /// page size and valid. - pub fn pfn(&self) -> PFN { - Into::<PFN>::into(self.raw_page) - } - - /// Returns the start physical address of the page, which is guaranteed to be - /// aligned to the page size and valid. - pub fn start(&self) -> PAddr { - PAddr::from(self.pfn()) - } - - /// Returns the physical address range of the page, which is guaranteed to be - /// aligned to the page size and valid. - pub fn range(&self) -> AddrRange<PAddr> { - AddrRange::from(self.start()).grow(self.len()) - } - - /// Get the allocator that manages this page. - pub fn allocator(&self) -> &A { - &self.alloc - } } -impl<A> Clone for Page<A> -where - A: PageAlloc, -{ - fn clone(&self) -> Self { - // SAFETY: Memory order here can be Relaxed is for the same reason as that - // in the copy constructor of `std::shared_ptr`. - self.raw_page.refcount().fetch_add(1, Ordering::Relaxed); - - Self { - raw_page: self.raw_page, - alloc: self.alloc.clone(), - } - } +/// A simple [`Folio`] with no reference counting or other ownership mechanism. +#[derive(Clone)] +pub struct BasicFolio { + pfn: PFN, + order: u32, } -impl<A> Drop for Page<A> -where - A: PageAlloc, -{ - fn drop(&mut self) { - match self.raw_page.refcount().fetch_sub(1, Ordering::AcqRel) { - 0 => panic!("Refcount for an in-use page is 0"), - 1 => unsafe { - // SAFETY: `self.raw_page` points to a valid page inside the global page array. - assert!(self.alloc.has_management_over(self.raw_page)); - - // SAFETY: `self.raw_page` is managed by the allocator and we're dropping the page. - self.alloc.dealloc(self.raw_page) - }, - _ => {} - } +impl BasicFolio { + pub const fn new(pfn: PFN, order: u32) -> Self { + Self { pfn, order } } } -impl<A: PageAlloc> fmt::Debug for Page<A> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "Page({:?}, order={})", - Into::<PFN>::into(self.raw_page), - self.order() - ) +impl Folio for BasicFolio { + fn pfn(&self) -> PFN { + self.pfn } -} -impl<T> PageAccess for T -where - T: PhysAccess, -{ - unsafe fn get_ptr_for_pfn(pfn: PFN) -> NonNull<PageBlock> { - unsafe { - // SAFETY: The physical address of a existing page must be - // aligned to the page size. - T::as_ptr(PAddr::from(pfn)) - } + fn order(&self) -> u32 { + self.order } } diff --git a/crates/eonix_mm/src/paging/page_alloc.rs b/crates/eonix_mm/src/paging/page_alloc.rs index fe222605..267d3ccb 100644 --- a/crates/eonix_mm/src/paging/page_alloc.rs +++ b/crates/eonix_mm/src/paging/page_alloc.rs @@ -1,89 +1,44 @@ -use super::{raw_page::UnmanagedRawPage, RawPage}; +use super::Folio; -/// A trait for allocating and deallocating pages of memory. +/// A trait for allocating and deallocating folios. /// /// Note that the instances of this trait should provide pointer-like or reference-like /// behavior, meaning that the allocators are to be passed around by value and stored in /// managed data structures. This is because the allocator may be used to deallocate the /// pages it allocates. -#[doc(notable_trait)] -pub trait PageAlloc: Clone { - type RawPage: RawPage; +pub trait FrameAlloc: Clone { + type Folio: Folio; - /// Allocate a page of the given *order*. - fn alloc_order(&self, order: u32) -> Option<Self::RawPage>; + /// Allocate a folio of the given *order*. + fn alloc_order(&self, order: u32) -> Option<Self::Folio>; - /// Allocate exactly one page. - fn alloc(&self) -> Option<Self::RawPage> { + /// Allocate exactly one folio. + fn alloc(&self) -> Option<Self::Folio> { self.alloc_order(0) } - /// Allocate a contiguous block of pages that can contain at least `count` pages. - fn alloc_at_least(&self, count: usize) -> Option<Self::RawPage> { + /// Allocate a folio that can contain at least [`count`] contiguous pages. + fn alloc_at_least(&self, count: usize) -> Option<Self::Folio> { let order = count.next_power_of_two().trailing_zeros(); self.alloc_order(order) } - - /// Deallocate a page. - /// - /// # Safety - /// This function is unsafe because it assumes that the caller MUST ensure that - /// `raw_page` is allocated in this allocator and never used after this call. - unsafe fn dealloc(&self, raw_page: Self::RawPage); - - /// Check whether the page is allocated and managed by the allocator. - fn has_management_over(&self, page_ptr: Self::RawPage) -> bool; } /// A trait for global page allocators. /// /// Global means that we can get an instance of the allocator from anywhere in the kernel. -#[doc(notable_trait)] -pub trait GlobalPageAlloc: PageAlloc + 'static { - /// Get the global page allocator. - fn global() -> Self; +pub trait GlobalFrameAlloc: FrameAlloc + 'static { + /// The global page allocator. + const GLOBAL: Self; } -#[derive(Clone)] -pub struct NoAlloc; - -impl<'a, A> PageAlloc for &'a A +impl<'a, A> FrameAlloc for &'a A where - A: PageAlloc, + A: FrameAlloc, { - type RawPage = A::RawPage; + type Folio = A::Folio; - fn alloc_order(&self, order: u32) -> Option<Self::RawPage> { + fn alloc_order(&self, order: u32) -> Option<Self::Folio> { (*self).alloc_order(order) } - - unsafe fn dealloc(&self, raw_page: Self::RawPage) { - unsafe { (*self).dealloc(raw_page) } - } - - fn has_management_over(&self, raw_page: Self::RawPage) -> bool { - (*self).has_management_over(raw_page) - } -} - -impl PageAlloc for NoAlloc { - type RawPage = UnmanagedRawPage; - - fn alloc_order(&self, _: u32) -> Option<Self::RawPage> { - panic!("`NoAlloc` cannot allocate pages"); - } - - unsafe fn dealloc(&self, _: Self::RawPage) { - panic!("`NoAlloc` cannot free pages"); - } - - fn has_management_over(&self, _: Self::RawPage) -> bool { - true - } -} - -impl GlobalPageAlloc for NoAlloc { - fn global() -> Self { - Self - } } diff --git a/crates/eonix_mm/src/paging/raw_page.rs b/crates/eonix_mm/src/paging/raw_page.rs deleted file mode 100644 index 7951729d..00000000 --- a/crates/eonix_mm/src/paging/raw_page.rs +++ /dev/null @@ -1,52 +0,0 @@ -use super::PFN; -use core::sync::atomic::AtomicUsize; - -/// A `RawPage` represents a page of memory in the kernel. It is a low-level -/// representation of a page that is used by the kernel to manage memory. -#[doc(notable_trait)] -pub trait RawPage: Clone + Copy + From<PFN> + Into<PFN> { - fn order(&self) -> u32; - fn refcount(&self) -> &AtomicUsize; - - fn is_present(&self) -> bool; -} - -#[derive(Clone, Copy)] -pub struct UnmanagedRawPage(PFN, u32); - -/// Unmanaged raw pages should always have a non-zero refcount to -/// avoid `free()` from being called. -static UNMANAGED_RAW_PAGE_CLONE_COUNT: AtomicUsize = AtomicUsize::new(1); - -impl UnmanagedRawPage { - pub const fn new(pfn: PFN, order: u32) -> Self { - Self(pfn, order) - } -} - -impl From<PFN> for UnmanagedRawPage { - fn from(value: PFN) -> Self { - Self::new(value, 0) - } -} - -impl Into<PFN> for UnmanagedRawPage { - fn into(self) -> PFN { - let Self(pfn, _) = self; - pfn - } -} - -impl RawPage for UnmanagedRawPage { - fn order(&self) -> u32 { - self.1 - } - - fn refcount(&self) -> &AtomicUsize { - &UNMANAGED_RAW_PAGE_CLONE_COUNT - } - - fn is_present(&self) -> bool { - true - } -} diff --git a/crates/eonix_mm/src/paging/zone.rs b/crates/eonix_mm/src/paging/zone.rs new file mode 100644 index 00000000..a2e85343 --- /dev/null +++ b/crates/eonix_mm/src/paging/zone.rs @@ -0,0 +1,19 @@ +use core::ptr::NonNull; + +use super::PFN; +use crate::address::PRange; + +/// A [`Zone`] holds a lot of [`Page`]s that share the same NUMA node or +/// "physical location". +pub trait Zone: Send + Sync { + type Page; + + /// Whether the [`range`] is within this [`Zone`]. + fn contains_prange(&self, range: PRange) -> bool; + + /// Get the [`RawPage`] that [`pfn`] points to. + /// + /// # Return + /// [`None`] if [`pfn`] is not in this [`Zone`]. + fn get_page(&self, pfn: PFN) -> Option<NonNull<Self::Page>>; +} diff --git a/crates/eonix_percpu/src/lib.rs b/crates/eonix_percpu/src/lib.rs index 1fc7ffb8..a00b5c05 100644 --- a/crates/eonix_percpu/src/lib.rs +++ b/crates/eonix_percpu/src/lib.rs @@ -1,28 +1,21 @@ #![no_std] use core::alloc::Layout; -use core::ptr::null_mut; -use core::ptr::NonNull; -use core::sync::atomic::AtomicPtr; -use core::sync::atomic::Ordering; - -#[cfg(target_arch = "x86_64")] -pub use eonix_percpu_macros::define_percpu_x86_64 as define_percpu; - -#[cfg(target_arch = "x86_64")] -pub use eonix_percpu_macros::define_percpu_shared_x86_64 as define_percpu_shared; - -#[cfg(target_arch = "riscv64")] -pub use eonix_percpu_macros::define_percpu_riscv64 as define_percpu; - -#[cfg(target_arch = "riscv64")] -pub use eonix_percpu_macros::define_percpu_shared_riscv64 as define_percpu_shared; +use core::ptr::{null_mut, NonNull}; +use core::sync::atomic::{AtomicPtr, Ordering}; #[cfg(target_arch = "loongarch64")] pub use eonix_percpu_macros::define_percpu_loongarch64 as define_percpu; - +#[cfg(target_arch = "riscv64")] +pub use eonix_percpu_macros::define_percpu_riscv64 as define_percpu; #[cfg(target_arch = "loongarch64")] pub use eonix_percpu_macros::define_percpu_shared_loongarch64 as define_percpu_shared; +#[cfg(target_arch = "riscv64")] +pub use eonix_percpu_macros::define_percpu_shared_riscv64 as define_percpu_shared; +#[cfg(target_arch = "x86_64")] +pub use eonix_percpu_macros::define_percpu_shared_x86_64 as define_percpu_shared; +#[cfg(target_arch = "x86_64")] +pub use eonix_percpu_macros::define_percpu_x86_64 as define_percpu; const MAX_CPUS: usize = 256; @@ -41,7 +34,7 @@ impl PercpuArea { unsafe extern "C" { fn PERCPU_LENGTH(); } - let len = PERCPU_LENGTH as usize; + let len = PERCPU_LENGTH as *const () as usize; assert_ne!(len, 0, "Percpu length should not be zero."); len @@ -52,7 +45,7 @@ impl PercpuArea { fn PERCPU_DATA_START(); } - let addr = PERCPU_DATA_START as usize; + let addr = PERCPU_DATA_START as *const () as usize; NonNull::new(addr as *mut _).expect("Percpu data should not be null.") } diff --git a/crates/eonix_runtime/src/context.rs b/crates/eonix_runtime/src/context.rs deleted file mode 100644 index 402d032f..00000000 --- a/crates/eonix_runtime/src/context.rs +++ /dev/null @@ -1,51 +0,0 @@ -use core::{cell::UnsafeCell, mem::transmute}; -use eonix_hal::context::TaskContext; -use eonix_hal::traits::context::RawTaskContext; - -#[derive(Debug)] -pub struct ExecutionContext(UnsafeCell<TaskContext>); - -unsafe impl Sync for ExecutionContext {} - -impl ExecutionContext { - pub const fn new() -> Self { - Self(UnsafeCell::new(TaskContext::new())) - } - - pub fn set_ip(&mut self, ip: usize) { - let Self(context) = self; - context.get_mut().set_program_counter(ip); - } - - pub fn set_sp(&mut self, sp: usize) { - let Self(context) = self; - context.get_mut().set_stack_pointer(sp); - } - - pub fn set_interrupt(&mut self, is_enabled: bool) { - let Self(context) = self; - context.get_mut().set_interrupt_enabled(is_enabled); - } - - pub fn call1<T>(&mut self, func: unsafe extern "C" fn(T) -> !, arg: usize) { - let Self(context) = self; - context - .get_mut() - .call(unsafe { transmute(func as *mut ()) }, arg); - } - - pub fn switch_to(&self, to: &Self) { - let Self(from_ctx) = self; - let Self(to_ctx) = to; - unsafe { - TaskContext::switch(&mut *from_ctx.get(), &mut *to_ctx.get()); - } - } - - pub fn switch_noreturn(&self) -> ! { - let Self(to_ctx) = self; - unsafe { - TaskContext::switch_to_noreturn(&mut *to_ctx.get()); - } - } -} diff --git a/crates/eonix_runtime/src/executor.rs b/crates/eonix_runtime/src/executor.rs index 3b858a47..fcb6ef29 100644 --- a/crates/eonix_runtime/src/executor.rs +++ b/crates/eonix_runtime/src/executor.rs @@ -1,27 +1,34 @@ -// mod builder; -mod output_handle; -mod stack; - -use alloc::{ - boxed::Box, - sync::{Arc, Weak}, -}; -use core::{ - marker::PhantomData, - pin::Pin, - task::{Context, Poll}, -}; -use eonix_sync::Spin; +use alloc::boxed::Box; +use alloc::sync::{Arc, Weak}; +use core::marker::PhantomData; +use core::pin::Pin; +use core::task::{Context, Poll, Waker}; -pub use output_handle::OutputHandle; -pub use stack::Stack; +use eonix_sync::Spin; /// An `Executor` executes a Future object in a separate thread of execution. /// -/// When the Future is finished, the `Executor` will call the `OutputHandle` to commit the output. -/// Then the `Executor` will release the resources associated with the Future. +/// When the Future is finished, the `Executor` will call the `OutputHandle` to +/// commit the output. Then the `Executor` will release the resources +/// associated with the Future. pub struct Executor(Option<Pin<Box<dyn TypeErasedExecutor>>>); +enum OutputState<Output> +where + Output: Send, +{ + Waiting(Option<Waker>), + Finished(Option<Output>), + TakenOut, +} + +pub struct OutputHandle<Output> +where + Output: Send, +{ + inner: OutputState<Output>, +} + trait TypeErasedExecutor: Send { fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()>; } @@ -92,3 +99,48 @@ impl Executor { } } } + +impl<Output> OutputHandle<Output> +where + Output: Send, +{ + pub fn new() -> Arc<Spin<Self>> { + Arc::new(Spin::new(Self { + inner: OutputState::Waiting(None), + })) + } + + pub fn try_resolve(&mut self) -> Option<Output> { + let output = match &mut self.inner { + OutputState::Waiting(_) => return None, + OutputState::Finished(output) => output.take(), + OutputState::TakenOut => panic!("Output already taken out"), + }; + + self.inner = OutputState::TakenOut; + if let Some(output) = output { + Some(output) + } else { + unreachable!("Output should be present") + } + } + + pub fn register_waiter(&mut self, waker: Waker) { + if let OutputState::Waiting(inner_waker) = &mut self.inner { + inner_waker.replace(waker); + } else { + panic!("Output is not waiting"); + } + } + + pub fn commit_output(&mut self, output: Output) { + if let OutputState::Waiting(inner_waker) = &mut self.inner { + if let Some(waker) = inner_waker.take() { + waker.wake(); + } + self.inner = OutputState::Finished(Some(output)); + } else { + panic!("Output is not waiting"); + } + } +} diff --git a/crates/eonix_runtime/src/executor/builder.rs b/crates/eonix_runtime/src/executor/builder.rs deleted file mode 100644 index 2729270b..00000000 --- a/crates/eonix_runtime/src/executor/builder.rs +++ /dev/null @@ -1,63 +0,0 @@ -use super::{Executor, OutputHandle, RealExecutor, Stack}; -use crate::context::ExecutionContext; -use alloc::{boxed::Box, sync::Arc}; -use core::{pin::Pin, sync::atomic::AtomicBool}; -use eonix_sync::Spin; - -pub struct ExecutorBuilder<S, R> { - stack: Option<S>, - runnable: Option<R>, -} - -impl<S, R> ExecutorBuilder<S, R> -where - S: Stack, - R::Output: Send, -{ - pub fn new() -> Self { - Self { - stack: None, - runnable: None, - } - } - - pub fn stack(mut self, stack: S) -> Self { - self.stack.replace(stack); - self - } - - pub fn runnable(mut self, runnable: R) -> Self { - self.runnable.replace(runnable); - self - } - - pub fn build( - mut self, - ) -> ( - Pin<Box<impl Executor>>, - ExecutionContext, - Arc<Spin<OutputHandle<R::Output>>>, - ) { - let stack = self.stack.take().expect("Stack is required"); - let runnable = self.runnable.take().expect("Runnable is required"); - - let mut execution_context = ExecutionContext::new(); - let output_handle = OutputHandle::new(); - - execution_context.set_sp(stack.get_bottom().addr().get() as _); - - let executor = Box::pin(RealExecutor { - _stack: stack, - runnable, - output_handle: Arc::downgrade(&output_handle), - finished: AtomicBool::new(false), - }); - - execution_context.call1( - RealExecutor::<S, R>::execute, - executor.as_ref().get_ref() as *const _ as usize, - ); - - (executor, execution_context, output_handle) - } -} diff --git a/crates/eonix_runtime/src/executor/output_handle.rs b/crates/eonix_runtime/src/executor/output_handle.rs deleted file mode 100644 index d0548fd2..00000000 --- a/crates/eonix_runtime/src/executor/output_handle.rs +++ /dev/null @@ -1,64 +0,0 @@ -use alloc::sync::Arc; -use core::task::Waker; -use eonix_sync::Spin; - -enum OutputState<Output> -where - Output: Send, -{ - Waiting(Option<Waker>), - Finished(Option<Output>), - TakenOut, -} - -pub struct OutputHandle<Output> -where - Output: Send, -{ - inner: OutputState<Output>, -} - -impl<Output> OutputHandle<Output> -where - Output: Send, -{ - pub fn new() -> Arc<Spin<Self>> { - Arc::new(Spin::new(Self { - inner: OutputState::Waiting(None), - })) - } - - pub fn try_resolve(&mut self) -> Option<Output> { - let output = match &mut self.inner { - OutputState::Waiting(_) => return None, - OutputState::Finished(output) => output.take(), - OutputState::TakenOut => panic!("Output already taken out"), - }; - - self.inner = OutputState::TakenOut; - if let Some(output) = output { - Some(output) - } else { - unreachable!("Output should be present") - } - } - - pub fn register_waiter(&mut self, waker: Waker) { - if let OutputState::Waiting(inner_waker) = &mut self.inner { - inner_waker.replace(waker); - } else { - panic!("Output is not waiting"); - } - } - - pub fn commit_output(&mut self, output: Output) { - if let OutputState::Waiting(inner_waker) = &mut self.inner { - if let Some(waker) = inner_waker.take() { - waker.wake(); - } - self.inner = OutputState::Finished(Some(output)); - } else { - panic!("Output is not waiting"); - } - } -} diff --git a/crates/eonix_runtime/src/executor/stack.rs b/crates/eonix_runtime/src/executor/stack.rs deleted file mode 100644 index cfd0825a..00000000 --- a/crates/eonix_runtime/src/executor/stack.rs +++ /dev/null @@ -1,6 +0,0 @@ -use core::ptr::NonNull; - -pub trait Stack: Sized + Send { - fn new() -> Self; - fn get_bottom(&self) -> NonNull<()>; -} diff --git a/crates/eonix_runtime/src/lib.rs b/crates/eonix_runtime/src/lib.rs index 458e01d9..c5da3014 100644 --- a/crates/eonix_runtime/src/lib.rs +++ b/crates/eonix_runtime/src/lib.rs @@ -1,6 +1,5 @@ #![no_std] -pub mod context; pub mod executor; mod ready_queue; pub mod scheduler; diff --git a/crates/eonix_runtime/src/scheduler.rs b/crates/eonix_runtime/src/scheduler.rs index 3f72fbf4..d658d579 100644 --- a/crates/eonix_runtime/src/scheduler.rs +++ b/crates/eonix_runtime/src/scheduler.rs @@ -1,20 +1,20 @@ -use crate::{ - executor::OutputHandle, - ready_queue::{local_rq, ReadyQueue}, - task::{Task, TaskAdapter, TaskHandle, TaskState}, -}; -use alloc::{sync::Arc, task::Wake}; -use core::{ - ops::{Deref, DerefMut}, - ptr::NonNull, - task::{Context, Poll, Waker}, -}; +use alloc::sync::Arc; +use alloc::task::Wake; +use core::hint::assert_unchecked; +use core::ops::{Deref, DerefMut}; +use core::ptr::NonNull; +use core::task::{Context, Poll, Waker}; + use eonix_hal::processor::halt; use eonix_log::println_trace; use eonix_sync::{LazyLock, Spin, SpinIrq as _}; use intrusive_collections::RBTree; use pointers::BorrowedArc; +use crate::executor::OutputHandle; +use crate::ready_queue::{local_rq, ReadyQueue}; +use crate::task::{Task, TaskAdapter, TaskHandle, TaskState}; + #[eonix_percpu::define_percpu] static CURRENT_TASK: Option<NonNull<Task>> = None; @@ -32,13 +32,30 @@ where impl Task { pub fn current<'a>() -> BorrowedArc<'a, Task> { unsafe { - // SAFETY: - // We should never "inspect" a change in `current`. - // The change of `CURRENT` will only happen in the scheduler. And if we are preempted, - // when we DO return, the `CURRENT` will be the same and remain valid. - BorrowedArc::from_raw(CURRENT_TASK.get().expect("Current task should be present")) + // SAFETY: We should never "inspect" a change in `current`: + // The change of `CURRENT` will only happen in the scheduler. + // And if we are preempted outside of the scheduler's context, when + // we return, `CURRENT` will be the same and remain valid. + BorrowedArc::from_raw( + CURRENT_TASK.get().expect("Current task should be present"), + ) } } + + #[inline(always)] + fn _swap_current(current: Option<Arc<Task>>) -> Option<Arc<Task>> { + let current_ptr = current.map(|arc| unsafe { + // SAFETY: `into_raw` always returns a valid pointer. + NonNull::new_unchecked(Arc::into_raw(arc) as *mut _) + }); + + let old = CURRENT_TASK.swap(current_ptr); + + old.map(|ptr| unsafe { + // SAFETY: The pointer always comes from Arc::into_raw + Arc::from_raw(ptr.as_ptr()) + }) + } } impl<O> JoinHandle<O> @@ -93,17 +110,10 @@ impl Runtime { } } - fn current(&self) -> Option<BorrowedArc<Task>> { - CURRENT_TASK - .get() - .map(|ptr| unsafe { BorrowedArc::from_raw(ptr) }) - } - - fn remove_and_enqueue_current(&self, rq: &mut impl DerefMut<Target = dyn ReadyQueue>) { - let Some(current) = CURRENT_TASK - .swap(None) - .map(|cur| unsafe { Arc::from_raw(cur.as_ptr()) }) - else { + fn remove_and_enqueue_current( + &self, rq: &mut impl DerefMut<Target = dyn ReadyQueue>, + ) { + let Some(current) = Task::_swap_current(None) else { return; }; @@ -116,7 +126,7 @@ impl Runtime { }) { Ok(TaskState::READY_RUNNING) => { println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Re-enqueueing task {:?} (CPU{})", current.id, eonix_hal::processor::CPU::local().cpuid(), @@ -126,7 +136,7 @@ impl Runtime { } Ok(_) => { println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Current task {:?} (CPU{}) is blocked, not re-enqueueing", current.id, eonix_hal::processor::CPU::local().cpuid(), @@ -136,7 +146,9 @@ impl Runtime { } } - pub fn block_till_woken(set_waker: impl FnOnce(&Waker)) -> impl Future<Output = ()> { + pub fn block_till_woken( + set_waker: impl FnOnce(&Waker), + ) -> impl Future<Output = ()> { struct BlockTillWoken<F: FnOnce(&Waker)> { set_waker: Option<F>, slept: bool, @@ -145,7 +157,9 @@ impl Runtime { impl<F: FnOnce(&Waker)> Future for BlockTillWoken<F> { type Output = (); - fn poll(self: core::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + fn poll( + self: core::pin::Pin<&mut Self>, cx: &mut Context<'_>, + ) -> Poll<()> { if self.slept { Poll::Ready(()) } else { @@ -184,7 +198,7 @@ impl Runtime { }; println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Switching to task {:?} (CPU{})", next.id, eonix_hal::processor::CPU::local().cpuid(), @@ -197,8 +211,10 @@ impl Runtime { "Next task should be in READY state" ); + let _old_current = Task::_swap_current(Some(next)); + debug_assert!(_old_current.is_none()); unsafe { - CURRENT_TASK.set(Some(NonNull::new_unchecked(Arc::into_raw(next) as *mut _))); + assert_unchecked(_old_current.is_none()); } drop(rq); @@ -212,14 +228,14 @@ impl Runtime { ); println_trace!( - "trace_scheduler", + feat: "trace_scheduler", "Task {:?} finished execution, removing...", Task::current().id, ); self.remove_task(&Task::current()); - CURRENT_TASK.set(None); + Task::_swap_current(None); } } } diff --git a/crates/eonix_sync/eonix_spin/src/lib.rs b/crates/eonix_sync/eonix_spin/src/lib.rs index 4718b867..7225aceb 100644 --- a/crates/eonix_sync/eonix_spin/src/lib.rs +++ b/crates/eonix_sync/eonix_spin/src/lib.rs @@ -2,13 +2,11 @@ mod guard; -use core::{ - cell::UnsafeCell, - marker::PhantomData, - sync::atomic::{AtomicBool, Ordering}, -}; -use eonix_sync_base::{Relax, SpinRelax}; +use core::cell::UnsafeCell; +use core::marker::PhantomData; +use core::sync::atomic::{AtomicBool, Ordering}; +use eonix_sync_base::{Relax, SpinRelax}; pub use guard::{SpinGuard, UnlockedSpinGuard}; pub trait SpinContext { @@ -84,7 +82,7 @@ where T: ?Sized, R: Relax, { - pub fn lock_with_context<C>(&self, context: C) -> SpinGuard<T, C, R> + pub fn lock_with_context<C>(&self, context: C) -> SpinGuard<'_, T, C, R> where C: SpinContext, { @@ -100,7 +98,7 @@ where ) } - pub fn lock(&self) -> SpinGuard<T, DisablePreemption, R> { + pub fn lock(&self) -> SpinGuard<'_, T, DisablePreemption, R> { self.lock_with_context(DisablePreemption::save()) } diff --git a/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs b/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs index 76a28682..b70cdc3d 100644 --- a/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs +++ b/crates/eonix_sync/eonix_sync_rt/src/spin_irq.rs @@ -12,7 +12,7 @@ pub trait SpinIrq { type Context: SpinContext; type Relax; - fn lock_irq(&self) -> SpinGuard<Self::Value, Self::Context, Self::Relax>; + fn lock_irq(&self) -> SpinGuard<'_, Self::Value, Self::Context, Self::Relax>; } impl SpinContext for IrqContext { @@ -50,7 +50,7 @@ where type Context = IrqContext; type Relax = R; - fn lock_irq(&self) -> SpinGuard<Self::Value, Self::Context, Self::Relax> { + fn lock_irq(&self) -> SpinGuard<'_, Self::Value, Self::Context, Self::Relax> { self.lock_with_context(IrqContext::save()) } } diff --git a/crates/intrusive_list/src/lib.rs b/crates/intrusive_list/src/lib.rs index af8c4f1a..440944d0 100644 --- a/crates/intrusive_list/src/lib.rs +++ b/crates/intrusive_list/src/lib.rs @@ -25,6 +25,8 @@ impl List { } pub fn insert(&mut self, node: &mut Link) { + // TODO: `node` above should be of 'static. + self.head.insert(node); self.count += 1; } diff --git a/crates/posix_types/src/open.rs b/crates/posix_types/src/open.rs index 758ea331..7135e5b8 100644 --- a/crates/posix_types/src/open.rs +++ b/crates/posix_types/src/open.rs @@ -11,6 +11,8 @@ bitflags! { const O_CREAT = 0x40; /// Exclusive access, fail if file exists const O_EXCL = 0x80; + /// Don't set controlling terminal. + const O_NOCTTY = 0x100; /// Truncate file to zero length if it exists const O_TRUNC = 0x200; /// Open file in append mode @@ -116,6 +118,8 @@ impl AtFlags { } pub fn statx_default_sync(&self) -> bool { - !self.intersects(AtFlags::AT_STATX_FORCE_SYNC | AtFlags::AT_STATX_DONT_SYNC) + !self.intersects( + AtFlags::AT_STATX_FORCE_SYNC | AtFlags::AT_STATX_DONT_SYNC, + ) } } diff --git a/crates/posix_types/src/poll.rs b/crates/posix_types/src/poll.rs index 781f589f..dcf5f9b2 100644 --- a/crates/posix_types/src/poll.rs +++ b/crates/posix_types/src/poll.rs @@ -1,5 +1,7 @@ pub const FDSET_LENGTH: usize = 1024 / (8 * size_of::<usize>()); +// TODO: Implement syscall pselect +#[allow(unused)] pub struct FDSet { fds_bits: [usize; FDSET_LENGTH], } diff --git a/crates/posix_types/src/result.rs b/crates/posix_types/src/result.rs index a10ff0ad..1535c444 100644 --- a/crates/posix_types/src/result.rs +++ b/crates/posix_types/src/result.rs @@ -1,14 +1,18 @@ pub enum PosixError { + ENOENT = 2, EFAULT = 14, EXDEV = 18, + ENOTDIR = 20, EINVAL = 22, } impl From<PosixError> for u32 { fn from(error: PosixError) -> Self { match error { + PosixError::ENOENT => 2, PosixError::EFAULT => 14, PosixError::EXDEV => 18, + PosixError::ENOTDIR => 20, PosixError::EINVAL => 22, } } @@ -17,8 +21,10 @@ impl From<PosixError> for u32 { impl core::fmt::Debug for PosixError { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { match self { + Self::ENOENT => write!(f, "ENOENT"), Self::EFAULT => write!(f, "EFAULT"), Self::EXDEV => write!(f, "EXDEV"), + Self::ENOTDIR => write!(f, "ENOTDIR"), Self::EINVAL => write!(f, "EINVAL"), } } diff --git a/crates/posix_types/src/syscall_no/loongarch64.rs b/crates/posix_types/src/syscall_no/loongarch64.rs index 19776a65..b0d54689 100644 --- a/crates/posix_types/src/syscall_no/loongarch64.rs +++ b/crates/posix_types/src/syscall_no/loongarch64.rs @@ -136,7 +136,7 @@ pub const SYS_RT_SIGSUSPEND: usize = 133; pub const SYS_RT_SIGACTION: usize = 134; pub const SYS_RT_SIGPROCMASK: usize = 135; pub const SYS_RT_SIGPENDING: usize = 136; -pub const SYS_RT_SIGTIMEDWAIT_TIME32: usize = 137; +pub const SYS_RT_SIGTIMEDWAIT: usize = 137; pub const SYS_RT_SIGQUEUEINFO: usize = 138; pub const SYS_RT_SIGRETURN: usize = 139; pub const SYS_SETPRIORITY: usize = 140; @@ -295,7 +295,6 @@ pub const SYS_RECVMMSG: usize = 417; pub const SYS_MQ_TIMEDSEND: usize = 418; pub const SYS_MQ_TIMEDRECEIVE: usize = 419; pub const SYS_SEMTIMEDOP: usize = 420; -pub const SYS_RT_SIGTIMEDWAIT: usize = 421; pub const SYS_FUTEX: usize = 422; pub const SYS_SCHED_RR_GET_INTERVAL: usize = 423; pub const SYS_PIDFD_SEND_SIGNAL: usize = 424; diff --git a/crates/posix_types/src/syscall_no/riscv64.rs b/crates/posix_types/src/syscall_no/riscv64.rs index 4457c20e..076942e9 100644 --- a/crates/posix_types/src/syscall_no/riscv64.rs +++ b/crates/posix_types/src/syscall_no/riscv64.rs @@ -136,7 +136,7 @@ pub const SYS_RT_SIGSUSPEND: usize = 133; pub const SYS_RT_SIGACTION: usize = 134; pub const SYS_RT_SIGPROCMASK: usize = 135; pub const SYS_RT_SIGPENDING: usize = 136; -pub const SYS_RT_SIGTIMEDWAIT_TIME32: usize = 137; +pub const SYS_RT_SIGTIMEDWAIT: usize = 137; pub const SYS_RT_SIGQUEUEINFO: usize = 138; pub const SYS_RT_SIGRETURN: usize = 139; pub const SYS_SETPRIORITY: usize = 140; diff --git a/crates/slab_allocator/Cargo.toml b/crates/slab_allocator/Cargo.toml index 067b6f53..926ac688 100644 --- a/crates/slab_allocator/Cargo.toml +++ b/crates/slab_allocator/Cargo.toml @@ -6,5 +6,3 @@ edition = "2024" [dependencies] eonix_mm = { path = "../eonix_mm" } eonix_sync = { path = "../eonix_sync" } -intrusive_list = { path = "../intrusive_list" } - diff --git a/crates/slab_allocator/src/debug.rs b/crates/slab_allocator/src/debug.rs new file mode 100644 index 00000000..e4169482 --- /dev/null +++ b/crates/slab_allocator/src/debug.rs @@ -0,0 +1,52 @@ +use core::mem::MaybeUninit; + +use eonix_mm::paging::FolioList; + +use crate::{SlabAlloc, SlabList, SlabPageAlloc}; + +pub struct SlabListStat { + pub object_size: usize, + + pub total_slabs: usize, + pub active_slabs: usize, + pub total_objects: usize, + pub active_objects: usize, +} + +impl<T> SlabList<T> +where + T: FolioList, +{ + pub fn dump_stats(&self) -> SlabListStat { + SlabListStat { + object_size: self.object_size, + total_slabs: self.total_folios, + active_slabs: self.active_folios, + total_objects: self.total_objects, + active_objects: self.active_objects, + } + } +} + +impl<P, const COUNT: usize> SlabAlloc<P, COUNT> +where + P: SlabPageAlloc, +{ + pub fn dump_stats(&self) -> [SlabListStat; COUNT] { + let mut stats: MaybeUninit<[SlabListStat; COUNT]> = + MaybeUninit::uninit(); + + for (i, slab_list) in self.slabs.iter().enumerate() { + let ptr: *mut SlabListStat = stats.as_mut_ptr().cast(); + + unsafe { + ptr.wrapping_add(i).write(slab_list.lock().dump_stats()); + } + } + + unsafe { + // SAFETY: All `COUNT` items are initialized. + stats.assume_init() + } + } +} diff --git a/crates/slab_allocator/src/lib.rs b/crates/slab_allocator/src/lib.rs index ce163183..1961f125 100644 --- a/crates/slab_allocator/src/lib.rs +++ b/crates/slab_allocator/src/lib.rs @@ -1,69 +1,321 @@ #![no_std] -mod slab_cache; +pub mod debug; -use core::{cmp::max, ptr::NonNull}; +use core::ptr::NonNull; -use eonix_mm::paging::{PageAlloc, RawPage}; +use eonix_mm::paging::{FolioList, FolioListSized}; use eonix_sync::Spin; -use intrusive_list::Link; -use slab_cache::SlabCache; -pub trait SlabRawPage: RawPage { - /// Get the container raw page struct of the list link. +#[repr(C)] +pub union SlabSlot { + slab_slot: Option<NonNull<SlabSlot>>, + data: u8, +} + +pub trait SlabPage: Sized + 'static { + fn get_data_ptr(&self) -> NonNull<[u8]>; + + fn get_free_slot(&self) -> Option<NonNull<SlabSlot>>; + fn set_free_slot(&mut self, next: Option<NonNull<SlabSlot>>); + + fn get_alloc_count(&self) -> usize; + + /// Increase the allocation count by 1 and return the increased value. + fn inc_alloc_count(&mut self) -> usize; + + /// Decrease the allocation count by 1 and return the decreased value. + fn dec_alloc_count(&mut self) -> usize; + + /// Get the [`SlabPage`] that `ptr` is allocated from. /// /// # Safety - /// The caller MUST ensure that the link points to a `RawPage`. - unsafe fn from_link(link: &mut Link) -> Self; + /// The caller MUST ensure that no others could be calling this function and + /// getting the [`SlabPage`] at the same time. + unsafe fn from_allocated(ptr: NonNull<u8>) -> &'static mut Self; +} + +pub(crate) trait SlabPageExt { + fn alloc_slot(&mut self) -> Option<NonNull<u8>>; - /// Get the list link of the raw page. - /// /// # Safety - /// The caller MUST ensure that at any time, only one mutable reference - /// to the link exists. - unsafe fn get_link(&self) -> &mut Link; + /// The caller MUST ensure that `slot_data_ptr` points to some position + /// previously allocated by [`SlabPageExt::alloc_slot`]. + unsafe fn free_slot(&mut self, slot_data_ptr: NonNull<u8>); - fn slab_init(&self, first_free: Option<NonNull<usize>>); + fn is_empty(&self) -> bool; + fn is_full(&self) -> bool; +} - // which slab page the ptr belong - fn in_which(ptr: *mut u8) -> Self; +impl<T> SlabPageExt for T +where + T: SlabPage, +{ + fn alloc_slot(&mut self) -> Option<NonNull<u8>> { + let mut free_slot = self.get_free_slot()?; - fn real_page_ptr(&self) -> *mut u8; + unsafe { + let free_slot = free_slot.as_mut(); - fn allocated_count(&self) -> &mut u32; + let next_slot = free_slot.slab_slot; + // ===== `free_slot` is now safe to be overwritten - fn next_free(&self) -> &mut Option<NonNull<usize>>; + self.set_free_slot(next_slot); + self.inc_alloc_count(); + + Some(NonNull::new_unchecked(&mut free_slot.data)) + } + } + + unsafe fn free_slot(&mut self, slot_data_ptr: NonNull<u8>) { + unsafe { + let mut free_slot: NonNull<SlabSlot> = slot_data_ptr.cast(); + free_slot.as_mut().slab_slot = self.get_free_slot(); + + self.set_free_slot(Some(free_slot)); + self.dec_alloc_count(); + } + } + + fn is_empty(&self) -> bool { + self.get_alloc_count() == 0 + } + + fn is_full(&self) -> bool { + self.get_free_slot().is_none() + } } -pub struct SlabAllocator<T, A, const SLAB_CACHE_COUNT: usize> { - slabs: [Spin<SlabCache<T, A>>; SLAB_CACHE_COUNT], - alloc: A, +/// Allocate a page suitable for slab system use. The page MUST come with +/// its allocation count 0 and next free slot None. +/// +/// # Safety +/// The page returned MUST have been properly initialized after allocation. +pub unsafe trait SlabPageAlloc { + type Page: SlabPage; + type PageList: FolioList<Folio = Self::Page>; + + fn alloc_slab_page(&self) -> &'static mut Self::Page; } -unsafe impl<T, A, const SLAB_CACHE_COUNT: usize> Send for SlabAllocator<T, A, SLAB_CACHE_COUNT> {} -unsafe impl<T, A, const SLAB_CACHE_COUNT: usize> Sync for SlabAllocator<T, A, SLAB_CACHE_COUNT> {} +pub(crate) struct SlabList<T> +where + T: FolioList, +{ + empty_list: T, + partial_list: T, + full_list: T, + object_size: usize, + + total_objects: usize, + active_objects: usize, + + total_folios: usize, + active_folios: usize, +} + +pub struct SlabAlloc<P, const COUNT: usize> +where + P: SlabPageAlloc, +{ + slabs: [Spin<SlabList<P::PageList>>; COUNT], + alloc: P, +} + +unsafe impl<P, const COUNT: usize> Send for SlabAlloc<P, COUNT> where + P: SlabPageAlloc +{ +} +unsafe impl<P, const COUNT: usize> Sync for SlabAlloc<P, COUNT> where + P: SlabPageAlloc +{ +} -impl<Raw, Allocator, const SLAB_CACHE_COUNT: usize> SlabAllocator<Raw, Allocator, SLAB_CACHE_COUNT> +impl<L, const COUNT: usize> SlabAlloc<L, COUNT> where - Raw: SlabRawPage, - Allocator: PageAlloc<RawPage = Raw>, + L: SlabPageAlloc, + L::PageList: FolioListSized, { - pub fn new_in(alloc: Allocator) -> Self { + pub fn new_in(alloc: L) -> Self { Self { - slabs: core::array::from_fn(|i| Spin::new(SlabCache::new_in(1 << (i + 3)))), + slabs: core::array::from_fn(|i| { + Spin::new(SlabList::new(1 << (i + 3))) + }), alloc, } } - pub fn alloc(&self, mut size: usize) -> *mut u8 { - size = max(8, size); + pub fn alloc(&self, mut size: usize) -> NonNull<u8> { + size = size.max(8); let idx = size.next_power_of_two().trailing_zeros() - 3; self.slabs[idx as usize].lock().alloc(&self.alloc) } - pub fn dealloc(&self, ptr: *mut u8, mut size: usize) { - size = max(8, size); + pub unsafe fn dealloc(&self, ptr: NonNull<u8>, mut size: usize) { + size = size.max(8); let idx = size.next_power_of_two().trailing_zeros() - 3; - self.slabs[idx as usize].lock().dealloc(ptr, &self.alloc); + + unsafe { + // SAFETY: + self.slabs[idx as usize].lock().dealloc(ptr, &self.alloc); + } + } +} + +impl<T> SlabList<T> +where + T: FolioListSized, +{ + const fn new(object_size: usize) -> Self { + Self { + empty_list: T::NEW, + partial_list: T::NEW, + full_list: T::NEW, + object_size, + total_objects: 0, + active_objects: 0, + total_folios: 0, + active_folios: 0, + } } } + +impl<T> SlabList<T> +where + T: FolioList, + T::Folio: SlabPage, +{ + fn alloc_from_partial(&mut self) -> NonNull<u8> { + let head = self.partial_list.peek_head().unwrap(); + let slot = head.alloc_slot().unwrap(); + + if head.is_full() { + let head = self.partial_list.pop_head().unwrap(); + self.full_list.push_tail(head); + } + + slot + } + + fn alloc_from_empty(&mut self) -> NonNull<u8> { + let head = self.empty_list.pop_head().unwrap(); + let slot = head.alloc_slot().unwrap(); + + if head.is_full() { + self.full_list.push_tail(head); + } else { + self.partial_list.push_tail(head); + } + + self.active_folios += 1; + + slot + } + + fn charge(&mut self, alloc: &impl SlabPageAlloc<Page = T::Folio>) { + let slab = alloc.alloc_slab_page(); + let free_slot = make_slab_page(slab.get_data_ptr(), self.object_size); + + slab.set_free_slot(Some(free_slot)); + + self.total_folios += 1; + self.total_objects += slab.get_data_ptr().len() / self.object_size; + + self.empty_list.push_tail(slab); + } + + fn alloc( + &mut self, alloc: &impl SlabPageAlloc<Page = T::Folio>, + ) -> NonNull<u8> { + // Increase active count here, since we won't fail for now... + self.active_objects += 1; + + if !self.partial_list.is_empty() { + return self.alloc_from_partial(); + } + + if self.empty_list.is_empty() { + self.charge(alloc); + } + + self.alloc_from_empty() + } + + unsafe fn dealloc( + &mut self, ptr: NonNull<u8>, _alloc: &impl SlabPageAlloc, + ) { + let slab_page = unsafe { + // SAFETY: + <T::Folio>::from_allocated(ptr) + }; + + let (was_full, is_empty); + + was_full = slab_page.is_full(); + + unsafe { + // SAFETY: + slab_page.free_slot(ptr); + } + + is_empty = slab_page.is_empty(); + + match (was_full, is_empty) { + (false, false) => {} + (false, true) => { + self.partial_list.remove(slab_page); + self.empty_list.push_tail(slab_page); + } + (true, false) => { + self.full_list.remove(slab_page); + self.partial_list.push_tail(slab_page); + } + (true, true) => { + self.full_list.remove(slab_page); + self.empty_list.push_tail(slab_page); + } + } + + if is_empty { + self.active_folios -= 1; + } + + self.active_objects -= 1; + + // TODO: Check whether we should place some pages back with `alloc` if + // the global free page count is below the watermark. + } +} + +fn make_slab_page( + page_ptr: NonNull<[u8]>, slot_size: usize, +) -> NonNull<SlabSlot> { + assert!( + slot_size >= core::mem::size_of::<usize>(), + "The minimum slot size is of a pointer's width" + ); + + let page_size = page_ptr.len(); + let slot_count = page_size / slot_size; + let page_start: NonNull<u8> = page_ptr.cast(); + + // Quick checks + assert!( + page_size % slot_size == 0, + "The page's size should be a multiple of the slot size" + ); + + let mut prev_free_slot = None; + for i in (0..slot_count).rev() { + let offset = i * slot_size; + + unsafe { + let mut slot_ptr: NonNull<SlabSlot> = page_start.add(offset).cast(); + + slot_ptr.as_mut().slab_slot = prev_free_slot; + prev_free_slot = Some(slot_ptr); + } + } + + prev_free_slot.expect("There should be at least one slot.") +} diff --git a/crates/slab_allocator/src/slab_cache.rs b/crates/slab_allocator/src/slab_cache.rs deleted file mode 100644 index 98e27fc8..00000000 --- a/crates/slab_allocator/src/slab_cache.rs +++ /dev/null @@ -1,164 +0,0 @@ -use super::SlabRawPage; -use core::{marker::PhantomData, ptr::NonNull}; -use eonix_mm::paging::{PageAlloc, PAGE_SIZE}; -use intrusive_list::List; - -pub(crate) struct SlabCache<T, A> { - empty_list: List, - partial_list: List, - full_list: List, - object_size: u32, - _phantom: PhantomData<(T, A)>, -} - -trait SlabRawPageExt { - fn alloc_slot(&self) -> Option<NonNull<usize>>; - fn dealloc_slot(&self, slot_ptr: *mut u8); - fn is_full(&self) -> bool; - fn is_empty(&self) -> bool; - fn slab_page_init(&self, object_size: u32) -> Option<NonNull<usize>>; -} - -impl<T> SlabRawPageExt for T -where - T: SlabRawPage, -{ - fn alloc_slot(&self) -> Option<NonNull<usize>> { - let ptr = self.next_free().clone(); - - let next_free = match ptr { - Some(ptr) => unsafe { ptr.read() as *mut usize }, - None => unreachable!(), - }; - *self.allocated_count() += 1; - *self.next_free() = NonNull::new(next_free); - return ptr; - } - - fn dealloc_slot(&self, slot_ptr: *mut u8) { - let slot_ptr = slot_ptr as *mut usize; - - if let Some(last_free) = self.next_free().clone() { - unsafe { *slot_ptr = last_free.as_ptr() as usize } - } else { - unsafe { *slot_ptr = 0 } - } - - *self.allocated_count() -= 1; - *self.next_free() = NonNull::new(slot_ptr); - } - - fn slab_page_init(&self, object_size: u32) -> Option<NonNull<usize>> { - assert!(object_size >= core::mem::size_of::<usize>() as u32); - - let first_free = self.real_page_ptr() as *mut usize; - - let mut slot_ptr = first_free; - let mut slot_count = PAGE_SIZE / object_size as usize; - - // SAFETY: carefully ptr operate - unsafe { - loop { - if slot_count == 1 { - *slot_ptr = 0; - break; - } - - let next_ptr = slot_ptr.byte_add(object_size as usize); - *slot_ptr = next_ptr as usize; - slot_ptr = next_ptr; - slot_count -= 1; - } - } - - NonNull::new(first_free) - } - - fn is_empty(&self) -> bool { - self.allocated_count().clone() == 0 - } - - fn is_full(&self) -> bool { - self.next_free().is_none() - } -} - -impl<Raw, Allocator> SlabCache<Raw, Allocator> -where - Raw: SlabRawPage, - Allocator: PageAlloc<RawPage = Raw>, -{ - pub(crate) const fn new_in(object_size: u32) -> Self { - // avoid unnecessary branch in alloc and dealloc - assert!(object_size <= PAGE_SIZE as u32 / 2); - - Self { - empty_list: List::new(), - partial_list: List::new(), - full_list: List::new(), - object_size: object_size, - _phantom: PhantomData, - } - } - - pub(crate) fn alloc(&mut self, alloc: &Allocator) -> *mut u8 { - if !self.partial_list.is_empty() { - let page_ptr = unsafe { - Raw::from_link( - self.partial_list - .head() - .expect("partial pages should not be empty"), - ) - }; - - let ptr = page_ptr.alloc_slot().expect("should get slot"); - - if page_ptr.is_full() { - self.partial_list.remove(unsafe { page_ptr.get_link() }); - self.full_list.insert(unsafe { page_ptr.get_link() }); - } - return ptr.as_ptr() as *mut u8; - } - - if !self.empty_list.is_empty() { - let page_ptr = unsafe { - Raw::from_link( - self.empty_list - .head() - .expect("empty pages should not be empty"), - ) - }; - - let ptr = page_ptr.alloc_slot().expect("should get slot"); - self.empty_list.remove(unsafe { page_ptr.get_link() }); - self.partial_list.insert(unsafe { page_ptr.get_link() }); - return ptr.as_ptr() as *mut u8; - } - - let new_page_ptr = alloc.alloc().expect("slab_cache get page fail!"); - let first_free = new_page_ptr.slab_page_init(self.object_size); - new_page_ptr.slab_init(first_free); - let ptr = new_page_ptr.alloc_slot().expect("should get slot"); - self.partial_list.insert(unsafe { new_page_ptr.get_link() }); - ptr.as_ptr() as *mut u8 - } - - pub(crate) fn dealloc(&mut self, ptr: *mut u8, _alloc: &Allocator) { - let page_ptr = Raw::in_which(ptr); - - if page_ptr.is_full() { - self.full_list.remove(unsafe { page_ptr.get_link() }); - self.partial_list.insert(unsafe { page_ptr.get_link() }); - } - - page_ptr.dealloc_slot(ptr); - - if page_ptr.is_empty() { - self.partial_list.remove(unsafe { page_ptr.get_link() }); - self.empty_list.insert(unsafe { page_ptr.get_link() }); - } - - // TODO: Check whether we should place some pages back with `alloc` if the global - // free page count is below the watermark. - } -} diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 722fa5da..42f72213 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -2,11 +2,13 @@ extern crate proc_macro; use proc_macro2::{Span, TokenStream}; use quote::quote; -use syn::{parse2, FnArg, Ident, ItemFn, LitStr}; +use syn::{parse2, FnArg, Ident, ItemFn, LitStr, ReturnType, Signature}; fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { if attrs.is_empty() { - panic!("`define_syscall` attribute should take one argument: `syscall_no`"); + panic!( + "`define_syscall` attribute should take one argument: `syscall_no`" + ); } let syscall_no = parse2::<Ident>(attrs).expect("Invalid syscall number"); @@ -37,37 +39,43 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { } }); - let args_call = item - .sig - .inputs - .iter() - .enumerate() - .map(|(idx, arg)| match arg { - FnArg::Receiver(_) => panic!("&self is not permitted."), - FnArg::Typed(_) => { - let arg_ident = Ident::new(&format!("arg_{}", idx), Span::call_site()); - quote! { #arg_ident } - } - }); + let args_call = + item.sig + .inputs + .iter() + .enumerate() + .map(|(idx, arg)| match arg { + FnArg::Receiver(_) => panic!("&self is not permitted."), + FnArg::Typed(_) => { + let arg_ident = + Ident::new(&format!("arg_{}", idx), Span::call_site()); + quote! { #arg_ident } + } + }); let syscall_name = item.sig.ident; - let syscall_name_str = LitStr::new(&syscall_name.to_string(), Span::call_site()); + let syscall_name_str = + LitStr::new(&syscall_name.to_string(), Span::call_site()); let body = item.block; - let helper_fn = Ident::new(&format!("_do_syscall_{}", syscall_name), Span::call_site()); + let helper_fn = + Ident::new(&format!("_do_syscall_{}", syscall_name), Span::call_site()); let helper_fn_pointer = Ident::new( &format!("_SYSCALL_ENTRY_{}", syscall_name.to_string().to_uppercase()), Span::call_site(), ); - let real_fn = Ident::new(&format!("sys_{}", syscall_name), Span::call_site()); + let real_fn = + Ident::new(&format!("sys_{}", syscall_name), Span::call_site()); let raw_syscall_section = LitStr::new( &format!(".raw_syscalls.{}", syscall_name), Span::call_site(), ); - let syscall_fn_section = - LitStr::new(&format!(".syscall_fns.{}", syscall_name), Span::call_site()); + let syscall_fn_section = LitStr::new( + &format!(".syscall_fns.{}", syscall_name), + Span::call_site(), + ); let trace_format_string = { let arg_count = item.sig.inputs.len(); @@ -84,7 +92,8 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { let args = args.enumerate().map(|(idx, arg)| match arg { FnArg::Receiver(_) => panic!("&self is not permitted."), FnArg::Typed(_) => { - let arg_ident = Ident::new(&format!("arg_{}", idx), Span::call_site()); + let arg_ident = + Ident::new(&format!("arg_{}", idx), Span::call_site()); quote! { #arg_ident } } }); @@ -123,7 +132,7 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { Box::new_in( async move { eonix_log::println_trace!( - "trace_syscall", + feat: "trace_syscall", "tid{}: {}({}) => {{", thd.tid, #syscall_name_str, @@ -133,7 +142,7 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { let retval = #real_fn(thd, #(#args_call),*).await.into_retval(); eonix_log::println_trace!( - "trace_syscall", + feat: "trace_syscall", "}} => {:x?}", retval, ); @@ -175,8 +184,116 @@ fn define_syscall_impl(attrs: TokenStream, item: TokenStream) -> TokenStream { /// ``` #[proc_macro_attribute] pub fn define_syscall( - attrs: proc_macro::TokenStream, - item: proc_macro::TokenStream, + attrs: proc_macro::TokenStream, item: proc_macro::TokenStream, ) -> proc_macro::TokenStream { define_syscall_impl(attrs.into(), item.into()).into() } + +fn check_late_init_sig(sig: &Signature) { + assert!(sig.constness.is_none(), "no const function allowed"); + assert!(sig.inputs.is_empty(), "no arguments allowed"); + + if let Some(abi) = &sig.abi { + if let Some(name) = &abi.name { + assert_eq!(name.value(), "Rust", "expected Rust ABI"); + } + } + + assert!( + matches!(&sig.output, ReturnType::Default), + "no return types allowed" + ); + + let generics = &sig.generics; + + assert_eq!(generics.const_params().count(), 0, "no generics allowed"); + assert_eq!(generics.type_params().count(), 0, "no generics allowed") +} + +fn define_late_init_normal( + func: &TokenStream, func_parsed: &ItemFn, +) -> TokenStream { + let func_ident = &func_parsed.sig.ident; + let func_name = func_parsed.sig.ident.to_string(); + + let static_ident = Ident::new( + &format!("__LATE_INIT_{}", func_name.to_uppercase()), + Span::call_site(), + ); + + quote! { + #func + + #[used] + #[doc(hidden)] + #[link_section = ".late_init"] + static #static_ident : fn() = #func_ident; + } +} + +fn define_late_init_async( + func: &TokenStream, func_parsed: &ItemFn, +) -> TokenStream { + let func_ident = &func_parsed.sig.ident; + let func_name = func_parsed.sig.ident.to_string(); + + let helper_ident = Ident::new( + &format!("__late_init_async_wrapper_{}", func_name.to_uppercase()), + Span::call_site(), + ); + + let static_ident = Ident::new( + &format!("__LATE_INIT_ASYNC_{}", func_name.to_uppercase()), + Span::call_site(), + ); + + let pin_t = quote!(core::pin::Pin); + let box_t = quote!(alloc::boxed::Box); + let future_t = quote!(core::future::Future); + + quote! { + #func + + fn #helper_ident() -> + #pin_t<#box_t<dyn #future_t<Output = ()> + Send>> { + #box_t::pin(#func_ident()) + } + + #[used] + #[doc(hidden)] + #[link_section = ".late_init_async"] + static #static_ident: fn() -> + #pin_t<#box_t<dyn #future_t<Output = ()> + Send>> = #helper_ident; + } +} + +fn define_late_init_impl(attrs: TokenStream, func: TokenStream) -> TokenStream { + if !attrs.is_empty() { + panic!("attributes not allowed here"); + } + + let func_parsed = + parse2::<ItemFn>(func.clone()).expect("expected function definition"); + + check_late_init_sig(&func_parsed.sig); + + if func_parsed.sig.asyncness.is_some() { + define_late_init_async(&func, &func_parsed) + } else { + define_late_init_normal(&func, &func_parsed) + } +} + +/// Define a function to run after kernel initialization. +/// +/// # Note +/// Keep in mind that all the functions defined with this macro may be run in +/// **ANY** order. +/// +/// So if order matters, try another way other than this! +#[proc_macro_attribute] +pub fn define_late_init( + attrs: proc_macro::TokenStream, func: proc_macro::TokenStream, +) -> proc_macro::TokenStream { + define_late_init_impl(attrs.into(), func.into()).into() +} diff --git a/rust-toolchain b/rust-toolchain index 8adb8e58..11ad5efd 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -nightly-2025-05-16 +nightly-2026-01-09 diff --git a/script/backtrace b/script/backtrace new file mode 100755 index 00000000..8a60c829 --- /dev/null +++ b/script/backtrace @@ -0,0 +1,100 @@ +#!/bin/bash + +ADDR2LINE=${ADDR2LINE:-riscv64-unknown-elf-addr2line} + +ksym=build/riscv64gc-unknown-none-elf/debug/eonix_kernel + +usage() { + cat <<EOF +Usage: $0 [options] [--] [addr2line options] + +Filter and translate backtrace addresses into human readable line numbers. + +Options: + -s, --ksym <symbol file> Use the given kernel symbol file + -o, --only-gbos Show kernel function calls only + -h, --help Show this message +EOF + exit "$1" +} + +# $1: instruction address +parse_pos() { + addr="$1" + shift + + "$ADDR2LINE" -e "$ksym" -i "$addr" "$@" 2>/dev/null +} + +filter_col() { + [ "$1" -eq 0 ] || awk "{ print \$$1; }" +} + +str_contains() { + grep -E "$1" >/dev/null 2>&1 +} + +filter_stacktrace() { + NL=$'\n' + _state=nonstart + _out= + while [ $_state != "end" ]; do + read -r _line + case $_state in + nonstart) + str_contains "8< CUT HERE" <<< "$_line" && _state=save + ;; + save) + if str_contains "8< CUT HERE" <<< "$_line"; then + _state=end + else + _out="$_out$_line$NL" + fi + ;; + esac + done + + echo "$_out" +} + +while [ "$#" -gt 0 ]; do + case "$1" in + -s|--ksym) + shift + ksym="$1" + ;; + -o|--only-gbos) + only_gb=y + ;; + --) + shift + break + ;; + -h|--help) + usage 0 + ;; + *) + usage 1 + ;; + esac + shift +done + +stacktrace="$(filter_stacktrace)" + +i=1 +for addr in $(filter_col 3 <<< "$stacktrace"); do + pos="$(parse_pos "$addr" "$@")" + + if [ -n "$only_gb" ]; then + if ! str_contains "greatbridf_os" <<< "$pos"; then + continue + fi + fi + + printf "========== %4d ==========\n" "$i" + + parse_pos "$addr" "$@" + + i=$((i + 1)) +done diff --git a/script/build-img.sh b/script/build-img.sh deleted file mode 100644 index 6a83638f..00000000 --- a/script/build-img.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh - -OS=`uname -s` - -if sudo --version > /dev/null 2>&1; then - SUDO=sudo -fi - -if [ "$OUTPUT" = "" ]; then - OUTPUT="build/fs-$ARCH.img" -fi - -if [ "$ARCH" = "" ]; then - echo "ARCH is not set, exiting..." >&2 - exit 1 -fi - -dd if=/dev/zero of="$OUTPUT" bs=`expr 1024 \* 1024` count=1020 -mkfs.fat -n SYSTEM "$OUTPUT" - -if [ "$OS" = "Darwin" ]; then - SUDO='' - hdiutil detach build/mnt > /dev/null 2>&1 || true - hdiutil attach "$OUTPUT" -mountpoint build/mnt -else - mkdir -p build/mnt - $SUDO losetup -P /dev/loop2 "$OUTPUT" - $SUDO mount /dev/loop2 build/mnt -fi - -if [ "$ARCH" = "x86_64" ]; then - $SUDO cp ./user-programs/init.out build/mnt/init - $SUDO cp ./user-programs/int.out build/mnt/int - $SUDO cp ./user-programs/dynamic_test build/mnt/dynamic_test - $SUDO cp ./user-programs/busybox build/mnt/busybox - $SUDO cp ./user-programs/busybox-minimal build/mnt/busybox_ - $SUDO cp ./user-programs/ld-musl-i386.so.1 build/mnt/ld-musl-i386.so.1 - $SUDO cp ./user-programs/pthread_test build/mnt/pthread_test - $SUDO cp ./user-programs/init_script_x86_64.sh build/mnt/initsh -elif [ "$ARCH" = "riscv64" ]; then - $SUDO cp ./user-programs/busybox.static build/mnt/busybox - $SUDO cp ./user-programs/init_script_riscv64.sh build/mnt/initsh -elif [ "$ARCH" = "loongarch64" ]; then - $SUDO cp ./user-programs/busybox.la64 build/mnt/busybox - $SUDO cp ./user-programs/init_script_loongarch64.sh build/mnt/initsh -fi - -# Add your custom files here - - -# End of custom files - -if [ "$OS" = "Darwin" ]; then - hdiutil detach build/mnt -else - $SUDO losetup -d /dev/loop2 - $SUDO umount build/mnt -fi diff --git a/script/build-img/build-img b/script/build-img/build-img new file mode 100755 index 00000000..638866af --- /dev/null +++ b/script/build-img/build-img @@ -0,0 +1,94 @@ +#!/bin/sh +# shellcheck source=./lib/lib.sh + +. "$(dirname "$(realpath "$0")")/lib/lib.sh" + +BASE_DIR="$(dirname "$(realpath "$0")")" + +KB=1024 +MB=$((1024 * KB)) + +TARGET= +IMAGE= +MOUNTPOINT="$(mktemp -d)" + +usage() { + cat >&2 <<EOF +Usage: $0 -t <TARGET> -o <OUTPUT> [-h] + +Options + -t The target building image for + [available: riscv64 x86 loongarch64] + -o Output image path + -h Show help message +EOF + + exit "$1" +} + +ensure_mount_macos() { + hdiutil detach "$MOUNTPOINT" > /dev/null 2>&1 || : + hdiutil attach "$IMAGE" -mountpoint "$MOUNTPOINT" +} + +ensure_mount_linux() { + DEV=$(sudo losetup -f "$IMAGE" --show) + sudo mount "$DEV" "$MOUNTPOINT" +} + +ensure_mount() { + if [ "$OS" = Darwin ]; then + ensure_mount_macos + return + fi + + ensure_mount_linux +} + +unmount_macos() { + hdiutil detach "$MOUNTPOINT" +} + +unmount_linux() { + sudo umount "$MOUNTPOINT" + sudo losetup -d "$DEV" +} + +cleanup() { + case "$OS" in + Darwin) + unmount_macos + ;; + *) + unmount_linux + ;; + esac +} + +set -eu + +while getopts "t:o:h" opt; do + case "$opt" in + t) TARGET="$OPTARG";; + o) IMAGE="$OPTARG";; + h) usage 0;; + ?) usage 1;; + esac +done + +shift $((OPTIND - 1)) + +[ -z "$TARGET" ] && die "ARCH is not set" +[ -z "$IMAGE" ] && die "output image path is not set" + +echo "Build image with ARCH=$TARGET OUTPUT=$IMAGE MOUNTPOINT=$MOUNTPOINT" + +dd if=/dev/zero of="$IMAGE" bs=$((1 * MB)) count=$((1024 - 4)) +mkfs.fat -n SYSTEM "$IMAGE" + +ensure_mount +trap cleanup EXIT + +for name in $(iter_files_sorted "$BASE_DIR/platform/$TARGET"); do + ( . "$name" ) +done diff --git a/script/build-img/lib/lib.sh b/script/build-img/lib/lib.sh new file mode 100644 index 00000000..5cc6c036 --- /dev/null +++ b/script/build-img/lib/lib.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +OS=$(uname -s) +SUDO=$(which sudo || :) + +info() { + echo "info : $1" +} + +warn() { + echo "warn : $1" >&2 +} + +error() { + echo "fatal: $1" >&2 +} + +die() { + error "$1" && exit 1 +} + +sudo() { + "$SUDO" "$@" +} + +copy_to_image() { + _prefix=sudo + [ "$OS" = Darwin ] && _prefix= + + $_prefix cp "$1" "$MOUNTPOINT/$2" +} + +iter_files() { + find -L "$1" -maxdepth 1 -type f +} + +iter_files_sorted() { + iter_files "$@" | sort +} diff --git a/script/build-img/platform/loongarch64-virt/10-base.sh b/script/build-img/platform/loongarch64-virt/10-base.sh new file mode 100644 index 00000000..d0f21dbd --- /dev/null +++ b/script/build-img/platform/loongarch64-virt/10-base.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# shellcheck source=./../../lib/lib.sh + +copy_to_image user-programs/busybox.la64 busybox +copy_to_image user-programs/init_script_loongarch64-virt.sh initsh diff --git a/script/build-img/platform/riscv64-virt/10-base.sh b/script/build-img/platform/riscv64-virt/10-base.sh new file mode 100644 index 00000000..efb32cf7 --- /dev/null +++ b/script/build-img/platform/riscv64-virt/10-base.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# shellcheck source=./../../lib/lib.sh + +copy_to_image user-programs/busybox.static busybox +copy_to_image user-programs/init_script_riscv64-virt.sh initsh diff --git a/script/build-img/platform/riscv64-virt/20-tests.sh b/script/build-img/platform/riscv64-virt/20-tests.sh new file mode 100644 index 00000000..1d9e0d71 --- /dev/null +++ b/script/build-img/platform/riscv64-virt/20-tests.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +name="$(mktemp)" + +cat > "$name" <<EOF +#!/bin/sh + +forks=100 + +getpid() { + echo "\$(exec sh -c 'echo \$PPID')" +} + +for i in \$(seq \$forks); do + ( getpid ) +done + +echo good +EOF + +copy_to_image "$name" test-fork diff --git a/script/build-img/platform/x86_64-acpi_compat/10-base.sh b/script/build-img/platform/x86_64-acpi_compat/10-base.sh new file mode 100644 index 00000000..e711f902 --- /dev/null +++ b/script/build-img/platform/x86_64-acpi_compat/10-base.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# shellcheck source=./../../lib/lib.sh + +copy_to_image user-programs/init.out init +copy_to_image user-programs/busybox busybox +copy_to_image user-programs/busybox-minimal busybox_ +copy_to_image user-programs/init_script_x86_64-acpi_compat.sh initsh diff --git a/script/build-img/platform/x86_64-acpi_compat/20-tests.sh b/script/build-img/platform/x86_64-acpi_compat/20-tests.sh new file mode 100644 index 00000000..69deed37 --- /dev/null +++ b/script/build-img/platform/x86_64-acpi_compat/20-tests.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# shellcheck source=./../../lib/lib.sh + +copy_to_image user-programs/int.out int +copy_to_image user-programs/dynamic_test dynamic_test +copy_to_image user-programs/ld-musl-i386.so.1 ld-musl-i386.so.1 +copy_to_image user-programs/pthread_test pthread_test diff --git a/script/test.sh b/script/test.sh index 01097bc3..cc5bde7e 100644 --- a/script/test.sh +++ b/script/test.sh @@ -1,13 +1,71 @@ -#!/bin/sh +#!/bin/bash +LOGFILE="build/test-$$.log" SUCCESS_MSG="###$RANDOM :SuCCeSS: $RANDOM###" -if [ "$ARCH" = "" ]; then - echo "Error: ARCH environment variable is not set." >&2 +die() { + echo "error: $1" >&2 exit 1 -fi +} + +runcmd() { + _cmd=$(printf "%q" "$*") + + cat <<EOF + expect { + "*/ # " { send "$_cmd\n" } + "*panicked" { exit 1 } + } +EOF +} + +wait_str() { + cat <<EOF + expect { + "*$*" {} + "*panicked" { exit 1 } + } +EOF +} + +wait_init_exit() { + echo 'expect "*init exited:" { exit 0 }' +} + +cleanup() { + killall -KILL "qemu-system-$ARCH" +} + +[ -z "$ARCH" ] && die "ARCH environment variable is not set" + +trap cleanup EXIT + +expect <<EOF | tee "$LOGFILE" + +set timeout 10 +spawn make test-run "ARCH=$ARCH" MODE=release "QEMU=qemu-system-$ARCH" -printf "ls\necho \"$SUCCESS_MSG\"\npoweroff\n" \ - | make test-run ARCH=$ARCH MODE=release QEMU=qemu-system-$ARCH \ - | tee build/test-$$.log \ - | grep "$SUCCESS_MSG" > /dev/null && echo TEST\ $$\ WITH\ ARCH=$ARCH\ PASSED +$(runcmd ls -la) + +$(wait_str proc) + +$(runcmd echo \""$SUCCESS_MSG"\") + +$(wait_str "\n$SUCCESS_MSG") + +$(runcmd poweroff) + +$(wait_init_exit) + +EOF + +status=$? + +echo + +# shellcheck disable=SC2181 +if [ $status -ne 0 ]; then + echo "=== Test $$ with ARCH=$ARCH failed" +else + echo "=== Test $$ with ARCH=$ARCH passed" +fi diff --git a/src/driver/ahci/command.rs b/src/driver/ahci/command.rs index c83339b7..4609d38d 100644 --- a/src/driver/ahci/command.rs +++ b/src/driver/ahci/command.rs @@ -1,9 +1,11 @@ +use eonix_mm::paging::Folio as _; + use crate::kernel::constants::EINVAL; -use crate::kernel::mem::paging::Page; +use crate::kernel::mem::Folio; use crate::prelude::*; pub trait Command { - fn pages(&self) -> &[Page]; + fn pages(&self) -> &[Folio]; fn lba(&self) -> u64; // in sectors @@ -14,19 +16,19 @@ pub trait Command { } pub struct IdentifyCommand { - page: Page, + page: Folio, } impl IdentifyCommand { pub fn new() -> Self { Self { - page: Page::alloc(), + page: Folio::alloc(), } } } impl Command for IdentifyCommand { - fn pages(&self) -> &[Page] { + fn pages(&self) -> &[Folio] { core::slice::from_ref(&self.page) } @@ -47,14 +49,14 @@ impl Command for IdentifyCommand { } } -pub struct ReadLBACommand<'lt> { - pages: &'lt [Page], +pub struct ReadLBACommand<'a> { + pages: &'a [Folio], lba: u64, count: u16, } -impl<'lt> ReadLBACommand<'lt> { - pub fn new(pages: &'lt [Page], lba: u64, count: u16) -> KResult<Self> { +impl<'a> ReadLBACommand<'a> { + pub fn new(pages: &'a [Folio], lba: u64, count: u16) -> KResult<Self> { if pages.len() > 248 { return Err(EINVAL); } @@ -69,7 +71,7 @@ impl<'lt> ReadLBACommand<'lt> { } impl Command for ReadLBACommand<'_> { - fn pages(&self) -> &[Page] { + fn pages(&self) -> &[Folio] { self.pages } @@ -91,13 +93,13 @@ impl Command for ReadLBACommand<'_> { } pub struct WriteLBACommand<'a> { - pages: &'a [Page], + pages: &'a [Folio], lba: u64, count: u16, } impl<'a> WriteLBACommand<'a> { - pub fn new(pages: &'a [Page], lba: u64, count: u16) -> KResult<Self> { + pub fn new(pages: &'a [Folio], lba: u64, count: u16) -> KResult<Self> { if pages.len() > 248 { return Err(EINVAL); } @@ -112,7 +114,7 @@ impl<'a> WriteLBACommand<'a> { } impl Command for WriteLBACommand<'_> { - fn pages(&self) -> &[Page] { + fn pages(&self) -> &[Folio] { self.pages } diff --git a/src/driver/ahci/command_table.rs b/src/driver/ahci/command_table.rs index c77b4abd..00fc8a0b 100644 --- a/src/driver/ahci/command_table.rs +++ b/src/driver/ahci/command_table.rs @@ -1,45 +1,55 @@ -use super::{command::Command, PRDTEntry, FISH2D}; -use crate::kernel::mem::{AsMemoryBlock as _, Page}; +use core::ptr::NonNull; + use eonix_mm::address::PAddr; +use eonix_mm::paging::Folio as _; -pub struct CommandTable<'a> { - page: Page, - command_fis: &'a mut FISH2D, +use super::command::Command; +use super::{PRDTEntry, FISH2D}; +use crate::kernel::mem::FolioOwned; - prdt: &'a mut [PRDTEntry; 248], - prdt_entries: Option<u16>, +pub struct CommandTable { + page: FolioOwned, + cmd_fis: NonNull<FISH2D>, + prdt: NonNull<[PRDTEntry; 248]>, + prdt_entries: usize, } -impl CommandTable<'_> { - pub fn new() -> Self { - let page = Page::alloc(); - let memory = page.as_memblk(); - - let (lhs, prdt) = memory.split_at(0x80); - - let (command_fis, _) = lhs.split_at(size_of::<FISH2D>()); - let command_fis = unsafe { command_fis.as_ptr().as_mut() }; - let prdt = unsafe { prdt.as_ptr().as_mut() }; +unsafe impl Send for CommandTable {} +unsafe impl Sync for CommandTable {} - Self { - page, - command_fis, - prdt, - prdt_entries: None, +impl CommandTable { + pub fn new() -> Self { + let page = FolioOwned::alloc(); + let base = page.get_ptr(); + + unsafe { + Self { + page, + cmd_fis: base.cast(), + prdt: base.byte_add(0x80).cast(), + prdt_entries: 0, + } } } pub fn setup(&mut self, cmd: &impl Command) { - self.command_fis.setup(cmd.cmd(), cmd.lba(), cmd.count()); - self.prdt_entries = Some(cmd.pages().len() as u16); + unsafe { + self.cmd_fis + .as_mut() + .setup(cmd.cmd(), cmd.lba(), cmd.count()); + } + + self.prdt_entries = cmd.pages().len(); for (idx, page) in cmd.pages().iter().enumerate() { - self.prdt[idx].setup(page); + unsafe { + self.prdt.as_mut()[idx].setup(page); + } } } - pub fn prdt_len(&self) -> u16 { - self.prdt_entries.unwrap() + pub fn prdt_len(&self) -> usize { + self.prdt_entries } pub fn base(&self) -> PAddr { diff --git a/src/driver/ahci/defs.rs b/src/driver/ahci/defs.rs index c5440246..66841da8 100644 --- a/src/driver/ahci/defs.rs +++ b/src/driver/ahci/defs.rs @@ -1,7 +1,9 @@ #![allow(dead_code)] -use crate::kernel::mem::paging::Page; use eonix_mm::address::Addr as _; +use eonix_mm::paging::Folio as _; + +use crate::kernel::mem::Folio; pub const VENDOR_INTEL: u16 = 0x8086; pub const DEVICE_AHCI: u16 = 0x2922; @@ -239,7 +241,7 @@ pub struct PRDTEntry { } impl PRDTEntry { - pub fn setup(&mut self, page: &Page) { + pub fn setup(&mut self, page: &Folio) { self.base = page.start().addr() as u64; self._reserved1 = 0; diff --git a/src/driver/ahci/mod.rs b/src/driver/ahci/mod.rs index c3b1cfa0..5fc83e3b 100644 --- a/src/driver/ahci/mod.rs +++ b/src/driver/ahci/mod.rs @@ -1,24 +1,24 @@ -use crate::{ - fs::procfs, - io::Buffer as _, - kernel::{ - block::{make_device, BlockDevice}, - constants::{EINVAL, EIO}, - interrupt::register_irq_handler, - pcie::{self, Header, PCIDevice, PCIDriver, PciError}, - task::block_on, - }, - prelude::*, -}; -use alloc::{format, sync::Arc}; +use alloc::format; +use alloc::sync::Arc; + +use async_trait::async_trait; use control::AdapterControl; use defs::*; +use eonix_log::println_warn; use eonix_mm::address::{AddrOps as _, PAddr}; use eonix_sync::SpinIrq as _; use port::AdapterPort; - pub(self) use register::Register; +use crate::fs::procfs; +use crate::io::Buffer as _; +use crate::kernel::block::BlockDevice; +use crate::kernel::constants::{EINVAL, EIO}; +use crate::kernel::interrupt::register_irq_handler; +use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; +use crate::kernel::vfs::types::DeviceId; +use crate::prelude::*; + mod command; mod command_table; mod control; @@ -29,7 +29,7 @@ pub(self) mod slot; mod stats; pub struct AHCIDriver { - devices: Spin<Vec<Arc<Device<'static>>>>, + devices: Spin<Vec<Arc<Device>>>, } pub struct BitsIterator { @@ -63,22 +63,22 @@ impl Iterator for BitsIterator { } } -struct Device<'a> { +struct Device { control_base: PAddr, control: AdapterControl, _pcidev: Arc<PCIDevice<'static>>, /// # Lock /// Might be accessed from irq handler, use with `lock_irq()` - ports: Spin<[Option<Arc<AdapterPort<'a>>>; 32]>, + ports: Spin<[Option<Arc<AdapterPort>>; 32]>, } /// # Safety /// `pcidev` is never accessed from Rust code /// TODO!!!: place *mut pci_device in a safe wrapper -unsafe impl Send for Device<'_> {} -unsafe impl Sync for Device<'_> {} +unsafe impl Send for Device {} +unsafe impl Sync for Device {} -impl Device<'_> { +impl Device { fn handle_interrupt(&self) { // Safety // `self.ports` is accessed inside irq handler @@ -107,8 +107,31 @@ impl Device<'_> { } } -impl Device<'static> { - fn probe_ports(&self) -> KResult<()> { +impl Device { + async fn probe_port(&self, port: Arc<AdapterPort>) -> KResult<()> { + port.init().await?; + + { + let port = port.clone(); + let name = format!("ahci-p{}-stats", port.nport); + procfs::populate_root(name.into_bytes().into(), move |buffer| { + port.print_stats(&mut buffer.get_writer()) + }) + .await; + } + + let port = BlockDevice::register_disk( + DeviceId::new(8, port.nport as u16 * 16), + 2147483647, // TODO: get size from device + port, + )?; + + port.partprobe().await?; + + Ok(()) + } + + async fn probe_ports(&self) -> KResult<()> { for nport in self.control.implemented_ports() { let port = Arc::new(AdapterPort::new(self.control_base, nport)); if !port.status_ok() { @@ -116,27 +139,7 @@ impl Device<'static> { } self.ports.lock_irq()[nport as usize] = Some(port.clone()); - if let Err(e) = (|| -> KResult<()> { - port.init()?; - - { - let port = port.clone(); - let name = format!("ahci-p{}-stats", port.nport); - procfs::populate_root(name.into_bytes().into(), move |buffer| { - port.print_stats(&mut buffer.get_writer()) - })?; - } - - let port = BlockDevice::register_disk( - make_device(8, nport * 16), - 2147483647, // TODO: get size from device - port, - )?; - - block_on(port.partprobe())?; - - Ok(()) - })() { + if let Err(e) = self.probe_port(port).await { self.ports.lock_irq()[nport as usize] = None; println_warn!("probe port {nport} failed with {e}"); } @@ -154,6 +157,7 @@ impl AHCIDriver { } } +#[async_trait] impl PCIDriver for AHCIDriver { fn vendor_id(&self) -> u16 { VENDOR_INTEL @@ -163,7 +167,9 @@ impl PCIDriver for AHCIDriver { DEVICE_AHCI } - fn handle_device(&self, pcidev: Arc<PCIDevice<'static>>) -> Result<(), PciError> { + async fn handle_device( + &self, pcidev: Arc<PCIDevice<'static>>, + ) -> Result<(), PciError> { let Header::Endpoint(header) = pcidev.header else { Err(EINVAL)? }; @@ -198,9 +204,11 @@ impl PCIDriver for AHCIDriver { device.control.enable_interrupts(); let device_irq = device.clone(); - register_irq_handler(irqno as i32, move || device_irq.handle_interrupt())?; + register_irq_handler(irqno as i32, move || { + device_irq.handle_interrupt() + })?; - device.probe_ports()?; + device.probe_ports().await?; self.devices.lock().push(device); @@ -208,6 +216,8 @@ impl PCIDriver for AHCIDriver { } } -pub fn register_ahci_driver() { - pcie::register_driver(AHCIDriver::new()).expect("Register ahci driver failed"); +pub async fn register_ahci_driver() { + pcie::register_driver(AHCIDriver::new()) + .await + .expect("Register ahci driver failed"); } diff --git a/src/driver/ahci/port.rs b/src/driver/ahci/port.rs index f558f6e1..a54bbbba 100644 --- a/src/driver/ahci/port.rs +++ b/src/driver/ahci/port.rs @@ -1,20 +1,18 @@ +use alloc::collections::vec_deque::VecDeque; +use core::task::{Poll, Waker}; + +use async_trait::async_trait; +use eonix_mm::address::{Addr as _, PAddr}; +use eonix_sync::SpinIrq as _; + use super::command::{Command, IdentifyCommand, ReadLBACommand, WriteLBACommand}; -use super::slot::CommandSlot; +use super::slot::CommandList; use super::stats::AdapterPortStats; -use super::{ - CommandHeader, Register, PORT_CMD_CR, PORT_CMD_FR, PORT_CMD_FRE, PORT_CMD_ST, PORT_IE_DEFAULT, -}; +use super::{Register, PORT_CMD_CR, PORT_CMD_FR, PORT_CMD_FRE, PORT_CMD_ST, PORT_IE_DEFAULT}; use crate::driver::ahci::command_table::CommandTable; use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; use crate::kernel::constants::{EINVAL, EIO}; -use crate::kernel::mem::paging::Page; -use crate::kernel::mem::AsMemoryBlock as _; -use crate::kernel::task::block_on; use crate::prelude::*; -use alloc::collections::vec_deque::VecDeque; -use core::pin::pin; -use eonix_mm::address::{Addr as _, PAddr}; -use eonix_sync::{SpinIrq as _, WaitList}; /// An `AdapterPort` is an HBA device in AHCI mode. /// @@ -55,6 +53,8 @@ pub struct AdapterPortData { struct FreeList { free: VecDeque<u32>, working: VecDeque<u32>, + + wakers: VecDeque<Waker>, } impl FreeList { @@ -62,57 +62,32 @@ impl FreeList { Self { free: (0..32).collect(), working: VecDeque::new(), + wakers: VecDeque::new(), } } } -pub struct AdapterPort<'a> { +pub struct AdapterPort { pub nport: u32, regs_base: PAddr, - slots: [CommandSlot<'a>; 32], + cmdlist: CommandList, free_list: Spin<FreeList>, - free_list_wait: WaitList, - - /// Holds the command list. - /// **DO NOT USE IT DIRECTLY** - _page: Page, - - cmdlist_base: PAddr, - fis_base: PAddr, stats: AdapterPortStats, } -impl<'a> AdapterPort<'a> { +impl AdapterPort { pub fn new(base: PAddr, nport: u32) -> Self { - let page = Page::alloc(); - let cmdlist_base = page.start(); - let cmdlist_size = 32 * size_of::<CommandHeader>(); - let fis_base = cmdlist_base + cmdlist_size; - - let (mut cmdheaders, _) = page.as_memblk().split_at(cmdlist_size); - let slots = core::array::from_fn(move |_| { - let (cmdheader, next) = cmdheaders.split_at(size_of::<CommandHeader>()); - cmdheaders = next; - CommandSlot::new(unsafe { cmdheader.as_ptr().as_mut() }) - }); - Self { nport, regs_base: base + 0x100 + 0x80 * nport as usize, - slots, + cmdlist: CommandList::new(), free_list: Spin::new(FreeList::new()), - free_list_wait: WaitList::new(), - _page: page, stats: AdapterPortStats::new(), - cmdlist_base, - fis_base, } } -} -impl AdapterPort<'_> { fn command_list_base(&self) -> Register<u64> { Register::new(self.regs_base + 0x00) } @@ -145,19 +120,17 @@ impl AdapterPort<'_> { self.sata_status().read_once() & 0xf == 0x3 } - fn get_free_slot(&self) -> u32 { - loop { + async fn get_free_slot(&self) -> u32 { + core::future::poll_fn(|ctx| { let mut free_list = self.free_list.lock_irq(); - let free_slot = free_list.free.pop_front(); - if let Some(slot) = free_slot { - return slot; + if let Some(slot) = free_list.free.pop_front() { + return Poll::Ready(slot); } - let mut wait = pin!(self.free_list_wait.prepare_to_wait()); - wait.as_mut().add_to_wait_list(); - drop(free_list); - block_on(wait); - } + free_list.wakers.push_back(ctx.waker().clone()); + Poll::Pending + }) + .await } fn save_working(&self, slot: u32) { @@ -165,8 +138,10 @@ impl AdapterPort<'_> { } fn release_free_slot(&self, slot: u32) { - self.free_list.lock_irq().free.push_back(slot); - self.free_list_wait.notify_one(); + let mut free_list = self.free_list.lock_irq(); + + free_list.free.push_back(slot); + free_list.wakers.drain(..).for_each(|waker| waker.wake()); } pub fn handle_interrupt(&self) { @@ -180,7 +155,7 @@ impl AdapterPort<'_> { return true; } - self.slots[n as usize].handle_irq(); + self.cmdlist.get(n as usize).handle_irq(); self.stats.inc_int_fired(); false @@ -204,12 +179,12 @@ impl AdapterPort<'_> { Ok(()) } - fn send_command(&self, cmd: &impl Command) -> KResult<()> { + async fn send_command(&self, cmd: &impl Command) -> KResult<()> { let mut cmdtable = CommandTable::new(); cmdtable.setup(cmd); - let slot_index = self.get_free_slot(); - let slot = &self.slots[slot_index as usize]; + let slot_index = self.get_free_slot().await; + let slot = self.cmdlist.get(slot_index as usize); slot.prepare_command(&cmdtable, cmd.write()); self.save_working(slot_index); @@ -222,36 +197,36 @@ impl AdapterPort<'_> { self.stats.inc_cmd_sent(); - if let Err(_) = block_on(slot.wait_finish()) { + slot.wait_finish().await.inspect_err(|_| { self.stats.inc_cmd_error(); - return Err(EIO); - }; + })?; self.release_free_slot(slot_index); Ok(()) } - fn identify(&self) -> KResult<()> { + async fn identify(&self) -> KResult<()> { let cmd = IdentifyCommand::new(); // TODO: check returned data - self.send_command(&cmd)?; + self.send_command(&cmd).await?; Ok(()) } - pub fn init(&self) -> KResult<()> { + pub async fn init(&self) -> KResult<()> { self.stop_command()?; self.command_list_base() - .write(self.cmdlist_base.addr() as u64); - self.fis_base().write(self.fis_base.addr() as u64); + .write(self.cmdlist.cmdlist_base().addr() as u64); + self.fis_base() + .write(self.cmdlist.recv_fis_base().addr() as u64); self.interrupt_enable().write_once(PORT_IE_DEFAULT); self.start_command()?; - match self.identify() { + match self.identify().await { Err(err) => { self.stop_command()?; Err(err) @@ -269,12 +244,13 @@ impl AdapterPort<'_> { } } -impl BlockRequestQueue for AdapterPort<'_> { +#[async_trait] +impl BlockRequestQueue for AdapterPort { fn max_request_pages(&self) -> u64 { 1024 } - fn submit(&self, req: BlockDeviceRequest) -> KResult<()> { + async fn submit<'a>(&'a self, req: BlockDeviceRequest<'a>) -> KResult<()> { match req { BlockDeviceRequest::Read { sector, @@ -287,7 +263,7 @@ impl BlockRequestQueue for AdapterPort<'_> { let command = ReadLBACommand::new(buffer, sector, count as u16)?; - self.send_command(&command) + self.send_command(&command).await } BlockDeviceRequest::Write { sector, @@ -300,7 +276,7 @@ impl BlockRequestQueue for AdapterPort<'_> { let command = WriteLBACommand::new(buffer, sector, count as u16)?; - self.send_command(&command) + self.send_command(&command).await } } } diff --git a/src/driver/ahci/slot.rs b/src/driver/ahci/slot.rs index 2198c457..fdb61f96 100644 --- a/src/driver/ahci/slot.rs +++ b/src/driver/ahci/slot.rs @@ -1,20 +1,37 @@ -use super::{command_table::CommandTable, CommandHeader}; +use core::cell::UnsafeCell; +use core::ptr::NonNull; +use core::task::{Poll, Waker}; + +use eonix_mm::address::{Addr as _, PAddr}; +use eonix_mm::paging::Folio as _; +use eonix_sync::{Spin, SpinIrq as _}; + +use super::command_table::CommandTable; +use super::CommandHeader; +use crate::kernel::constants::EIO; +use crate::kernel::mem::FolioOwned; use crate::KResult; -use core::pin::pin; -use eonix_mm::address::Addr as _; -use eonix_sync::{Spin, SpinIrq as _, WaitList}; + +pub struct CommandList { + base: NonNull<u8>, + _page: FolioOwned, +} + +unsafe impl Send for CommandList {} +unsafe impl Sync for CommandList {} pub struct CommandSlot<'a> { - /// # Usage - /// `inner.cmdheader` might be used in irq handler. So in order to wait for - /// commands to finish, we should use `lock_irq` on `inner` - inner: Spin<CommandSlotInner<'a>>, - wait_list: WaitList, + cmdheader: &'a UnsafeCell<CommandHeader>, + /// [`Self::control`] might be used in irq handlers. + control: &'a Spin<SlotControl>, } -struct CommandSlotInner<'a> { +unsafe impl Send for CommandSlot<'_> {} +unsafe impl Sync for CommandSlot<'_> {} + +struct SlotControl { state: SlotState, - cmdheader: &'a mut CommandHeader, + waker: Option<Waker>, } #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -22,35 +39,110 @@ enum SlotState { Idle, Working, Finished, + // TODO: Implement AHCI error handling + #[allow(unused)] Error, } -impl<'a> CommandSlot<'a> { - pub fn new(cmdheader: &'a mut CommandHeader) -> Self { +impl CommandList { + fn cmdheaders(&self) -> &[UnsafeCell<CommandHeader>; 32] { + unsafe { self.base.cast().as_ref() } + } + + fn controls_ptr(base: NonNull<u8>) -> NonNull<Spin<SlotControl>> { + // 24 bytes for SlotControl and extra 8 bytes for Spin. + const_assert_eq!(size_of::<Spin<SlotControl>>(), 32); + + unsafe { base.add(size_of::<UnsafeCell<CommandHeader>>() * 32).cast() } + } + + fn controls(&self) -> &[Spin<SlotControl>; 32] { + unsafe { Self::controls_ptr(self.base).cast().as_ref() } + } + + pub fn cmdlist_base(&self) -> PAddr { + self._page.start() + } + + pub fn recv_fis_base(&self) -> PAddr { + self._page.start() + + (size_of::<UnsafeCell<CommandHeader>>() + size_of::<Spin<SlotControl>>()) * 32 + } + + pub fn get(&self, index: usize) -> CommandSlot<'_> { + CommandSlot { + cmdheader: &self.cmdheaders()[index], + control: &self.controls()[index], + } + } + + pub fn new() -> Self { + let mut page = FolioOwned::alloc(); + page.as_bytes_mut().fill(0); + + let base = page.get_ptr(); + + let controls_ptr = Self::controls_ptr(base); + + for i in 0..32 { + unsafe { + controls_ptr.add(i).write(Spin::new(SlotControl { + state: SlotState::Idle, + waker: None, + })); + } + } + Self { - inner: Spin::new(CommandSlotInner { - state: SlotState::Idle, - cmdheader, - }), - wait_list: WaitList::new(), + base: page.get_ptr(), + _page: page, } } +} +impl Drop for CommandList { + fn drop(&mut self) { + let controls_ptr = Self::controls_ptr(self.base); + + for i in 0..32 { + unsafe { + controls_ptr.add(i).drop_in_place(); + } + } + } +} + +impl CommandSlot<'_> { pub fn handle_irq(&self) { - let mut inner = self.inner.lock(); - debug_assert_eq!(inner.state, SlotState::Working); + // We are already in the IRQ handler. + let mut control = self.control.lock(); + assert_eq!(control.state, SlotState::Working); + + let cmdheader = unsafe { + // SAFETY: The IRQ handler is only called after the command + // is finished. + &mut *self.cmdheader.get() + }; // TODO: Check errors. - inner.state = SlotState::Finished; - inner.cmdheader.bytes_transferred = 0; - inner.cmdheader.prdt_length = 0; + cmdheader.bytes_transferred = 0; + cmdheader.prdt_length = 0; - self.wait_list.notify_all(); + control.state = SlotState::Finished; + + if let Some(waker) = control.waker.take() { + waker.wake(); + } } pub fn prepare_command(&self, cmdtable: &CommandTable, write: bool) { - let mut inner = self.inner.lock_irq(); - let cmdheader = &mut inner.cmdheader; + let mut control = self.control.lock_irq(); + assert_eq!(control.state, SlotState::Idle); + + let cmdheader = unsafe { + // SAFETY: We are in the idle state. + &mut *self.cmdheader.get() + }; cmdheader.first = 0x05; // FIS type @@ -60,35 +152,37 @@ impl<'a> CommandSlot<'a> { cmdheader.second = 0x00; - cmdheader.prdt_length = cmdtable.prdt_len(); + cmdheader.prdt_length = cmdtable.prdt_len() as u16; cmdheader.bytes_transferred = 0; cmdheader.command_table_base = cmdtable.base().addr() as u64; cmdheader._reserved = [0; 4]; - inner.state = SlotState::Working; + control.state = SlotState::Working; } pub async fn wait_finish(&self) -> KResult<()> { - let mut inner = loop { - let inner = self.inner.lock_irq(); - if inner.state != SlotState::Working { - break inner; + core::future::poll_fn(|ctx| { + let mut control = self.control.lock_irq(); + + match control.state { + SlotState::Idle => unreachable!("Poll called in idle state"), + SlotState::Working => { + control.waker = Some(ctx.waker().clone()); + Poll::Pending + } + SlotState::Finished => { + control.state = SlotState::Idle; + Poll::Ready(Ok(())) + } + SlotState::Error => { + control.state = SlotState::Idle; + + // TODO: Report errors. + Poll::Ready(Err(EIO)) + } } - - let mut wait = pin!(self.wait_list.prepare_to_wait()); - wait.as_mut().add_to_wait_list(); - - if inner.state != SlotState::Working { - break inner; - } - - drop(inner); - wait.await; - }; - - inner.state = SlotState::Idle; - - Ok(()) + }) + .await } } diff --git a/src/driver/e1000e.rs b/src/driver/e1000e.rs index ba31b8b1..079b29ae 100644 --- a/src/driver/e1000e.rs +++ b/src/driver/e1000e.rs @@ -1,18 +1,20 @@ -use crate::kernel::constants::{EAGAIN, EFAULT, EINVAL, EIO}; -use crate::kernel::interrupt::register_irq_handler; -use crate::kernel::mem::paging::{self, AllocZeroed}; -use crate::kernel::mem::{AsMemoryBlock, PhysAccess}; -use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; -use crate::net::netdev; -use crate::prelude::*; use alloc::boxed::Box; use alloc::sync::Arc; -use alloc::vec::Vec; use core::ptr::NonNull; + +use async_trait::async_trait; use eonix_hal::fence::memory_barrier; +use eonix_log::println_debug; use eonix_mm::address::{Addr, PAddr}; +use eonix_mm::paging::Folio as _; use eonix_sync::SpinIrq; -use paging::Page; + +use crate::kernel::constants::{EAGAIN, EFAULT, EINVAL, EIO}; +use crate::kernel::interrupt::register_irq_handler; +use crate::kernel::mem::{FolioOwned, PhysAccess}; +use crate::kernel::pcie::{self, Header, PCIDevice, PCIDriver, PciError}; +use crate::net::netdev; +use crate::prelude::*; mod defs; @@ -54,13 +56,15 @@ struct E1000eDev { id: u32, regs: Registers, - rt_desc_page: Page, + rt_desc_page: FolioOwned, rx_head: Option<u32>, rx_tail: Option<u32>, tx_tail: Option<u32>, - rx_buffers: Option<Box<Vec<Page>>>, - tx_buffers: Option<Box<Vec<Page>>>, + rx_buffers: Box<[FolioOwned; RX_DESC_SIZE]>, + // TODO: Implement E1000e send + #[allow(unused)] + tx_buffers: Box<[Option<FolioOwned>; TX_DESC_SIZE]>, } fn test(val: u32, bit: u32) -> bool { @@ -149,7 +153,9 @@ impl netdev::Netdev for E1000eDev { match status & defs::STAT_SPEED_MASK { defs::STAT_SPEED_10M => self.speed = netdev::LinkSpeed::Speed10M, defs::STAT_SPEED_100M => self.speed = netdev::LinkSpeed::Speed100M, - defs::STAT_SPEED_1000M => self.speed = netdev::LinkSpeed::Speed1000M, + defs::STAT_SPEED_1000M => { + self.speed = netdev::LinkSpeed::Speed1000M + } _ => return Err(EINVAL), } @@ -195,7 +201,7 @@ impl netdev::Netdev for E1000eDev { break; } - let ref mut desc = self.rx_desc_table()[next_tail as usize]; + let desc = unsafe { &mut self.rx_desc_table()[next_tail as usize] }; if !test(desc.status as u32, defs::RXD_STAT_DD as u32) { Err(EIO)?; } @@ -203,13 +209,13 @@ impl netdev::Netdev for E1000eDev { desc.status = 0; let len = desc.length as usize; - let buffers = self.rx_buffers.as_mut().ok_or(EIO)?; - let data = unsafe { - // SAFETY: No one could be writing to the buffer at this point. - &buffers[next_tail as usize].as_memblk().as_bytes()[..len] - }; + let buffer = &self.rx_buffers[next_tail as usize]; + let data = &buffer.as_bytes()[..len]; - println_debug!("e1000e: received {len} bytes, {:?}", PrintableBytes(data)); + println_debug!( + "e1000e: received {len} bytes, {:?}", + PrintableBytes(data) + ); self.rx_tail = Some(next_tail); } @@ -225,20 +231,17 @@ impl netdev::Netdev for E1000eDev { return Err(EAGAIN); } - let ref mut desc = self.tx_desc_table()[tail as usize]; + let desc = unsafe { &mut self.tx_desc_table()[tail as usize] }; if !test(desc.status as u32, defs::TXD_STAT_DD as u32) { return Err(EIO); } - let buffer_page = Page::alloc(); + let mut buffer_page = FolioOwned::alloc(); if buf.len() > buffer_page.len() { return Err(EFAULT); } - unsafe { - // SAFETY: We are the only one writing to this memory block. - buffer_page.as_memblk().as_bytes_mut()[..buf.len()].copy_from_slice(buf); - } + buffer_page.as_bytes_mut()[..buf.len()].copy_from_slice(buf); desc.buffer = PAddr::from(buffer_page.pfn()).addr() as u64; desc.length = buf.len() as u16; @@ -248,9 +251,10 @@ impl netdev::Netdev for E1000eDev { self.tx_tail = Some(next_tail); self.regs.write(defs::REG_TDT, next_tail); - // TODO: check if the packets are sent and update self.tx_head state - - Ok(()) + unimplemented!( + "Check if the packets are sent and update self.tx_head state" + ); + // Ok(()) } } @@ -323,26 +327,26 @@ impl E1000eDev { Ok(()) } - fn reset(&self) -> Result<(), u32> { + fn reset(regs: &Registers) -> Result<(), u32> { // disable interrupts so we won't mess things up - self.regs.write(defs::REG_IMC, 0xffffffff); + regs.write(defs::REG_IMC, 0xffffffff); - let ctrl = self.regs.read(defs::REG_CTRL); - self.regs.write(defs::REG_CTRL, ctrl | defs::CTRL_GIOD); + let ctrl = regs.read(defs::REG_CTRL); + regs.write(defs::REG_CTRL, ctrl | defs::CTRL_GIOD); - while self.regs.read(defs::REG_STAT) & defs::STAT_GIOE != 0 { + while regs.read(defs::REG_STAT) & defs::STAT_GIOE != 0 { // wait for link up } - let ctrl = self.regs.read(defs::REG_CTRL); - self.regs.write(defs::REG_CTRL, ctrl | defs::CTRL_RST); + let ctrl = regs.read(defs::REG_CTRL); + regs.write(defs::REG_CTRL, ctrl | defs::CTRL_RST); - while self.regs.read(defs::REG_CTRL) & defs::CTRL_RST != 0 { + while regs.read(defs::REG_CTRL) & defs::CTRL_RST != 0 { // wait for reset } // disable interrupts again - self.regs.write(defs::REG_IMC, 0xffffffff); + regs.write(defs::REG_IMC, 0xffffffff); Ok(()) } @@ -359,64 +363,53 @@ impl E1000eDev { Ok(()) } - pub fn new(base: PAddr, irq_no: usize) -> Result<Self, u32> { - let page = Page::zeroed(); + pub fn new(base: PAddr, irq_no: usize) -> KResult<Self> { + let regs = Registers::new(base); + Self::reset(®s)?; - let mut dev = Self { + let dev = Self { irq_no, - mac: [0; 6], + mac: regs.read_as(0x5400), status: netdev::LinkStatus::Down, speed: netdev::LinkSpeed::SpeedUnknown, id: netdev::alloc_id(), - regs: Registers::new(base), - rt_desc_page: page, + regs, + rt_desc_page: { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + folio + }, rx_head: None, rx_tail: None, tx_tail: None, - rx_buffers: None, - tx_buffers: None, + rx_buffers: Box::new(core::array::from_fn(|_| { + FolioOwned::alloc_order(2) + })), + tx_buffers: Box::new([const { None }; 32]), }; - dev.reset()?; - - dev.mac = dev.regs.read_as(0x5400); - dev.tx_buffers = Some(Box::new(Vec::with_capacity(TX_DESC_SIZE))); - - let mut rx_buffers = Box::new(Vec::with_capacity(RX_DESC_SIZE)); - - for index in 0..RX_DESC_SIZE { - let page = Page::alloc_order(2); - - let ref mut desc = dev.rx_desc_table()[index]; - desc.buffer = PAddr::from(page.pfn()).addr() as u64; - desc.status = 0; - - rx_buffers.push(page); - } + unsafe { + for (desc, page) in + dev.rx_desc_table().into_iter().zip(dev.rx_buffers.iter()) + { + desc.buffer = page.start().addr() as u64; + desc.status = 0; + } - for index in 0..TX_DESC_SIZE { - let ref mut desc = dev.tx_desc_table()[index]; - desc.status = defs::TXD_STAT_DD; + for desc in dev.tx_desc_table() { + desc.status = defs::TXD_STAT_DD; + } } - dev.rx_buffers = Some(rx_buffers); - Ok(dev) } - fn rx_desc_table(&self) -> &mut [RxDescriptor; RX_DESC_SIZE] { - unsafe { - // SAFETY: TODO - self.rt_desc_page.as_memblk().as_ptr().as_mut() - } + unsafe fn rx_desc_table(&self) -> &mut [RxDescriptor; RX_DESC_SIZE] { + self.rt_desc_page.get_ptr().cast().as_mut() } - fn tx_desc_table(&self) -> &mut [TxDescriptor; TX_DESC_SIZE] { - let (_, right) = self.rt_desc_page.as_memblk().split_at(0x200); - unsafe { - // SAFETY: TODO - right.as_ptr().as_mut() - } + unsafe fn tx_desc_table(&self) -> &mut [TxDescriptor; TX_DESC_SIZE] { + self.rt_desc_page.get_ptr().add(0x200).cast().as_mut() } } @@ -424,12 +417,8 @@ impl Drop for E1000eDev { fn drop(&mut self) { assert_eq!(self.status, netdev::LinkStatus::Down); - if let Some(_) = self.rx_buffers.take() {} - - // TODO: we should wait until all packets are sent - if let Some(_) = self.tx_buffers.take() {} - - let _ = self.rt_desc_page; + // TODO: we should wait until all packets are sent before dropping + // tx buffers. } } @@ -437,6 +426,7 @@ struct Driver { dev_id: u16, } +#[async_trait] impl PCIDriver for Driver { fn vendor_id(&self) -> u16 { 0x8086 @@ -446,7 +436,9 @@ impl PCIDriver for Driver { self.dev_id } - fn handle_device(&self, device: Arc<PCIDevice<'static>>) -> Result<(), PciError> { + async fn handle_device( + &self, device: Arc<PCIDevice<'static>>, + ) -> Result<(), PciError> { let Header::Endpoint(header) = device.header else { Err(EINVAL)? }; @@ -473,10 +465,10 @@ impl PCIDriver for Driver { } } -pub fn register_e1000e_driver() { +pub async fn register_e1000e_driver() { let dev_ids = [0x100e, 0x10d3, 0x10ea, 0x153a]; for id in dev_ids.into_iter() { - pcie::register_driver(Driver { dev_id: id }).unwrap(); + pcie::register_driver(Driver { dev_id: id }).await.unwrap(); } } diff --git a/src/driver/serial.rs b/src/driver/serial.rs index d69965f4..b84a1c5f 100644 --- a/src/driver/serial.rs +++ b/src/driver/serial.rs @@ -2,8 +2,8 @@ mod io; use crate::{ kernel::{ - block::make_device, console::set_console, constants::EIO, interrupt::register_irq_handler, - CharDevice, CharDeviceType, Terminal, TerminalDevice, + console::set_console, constants::EIO, interrupt::register_irq_handler, + vfs::types::DeviceId, CharDevice, CharDeviceType, Terminal, TerminalDevice, }, prelude::*, }; @@ -167,7 +167,7 @@ impl Serial { eonix_log::set_console(terminal.clone()); CharDevice::register( - make_device(4, 64 + port.id), + DeviceId::new(4, 64 + port.id as u16), port.name.clone(), CharDeviceType::Terminal(terminal), )?; @@ -249,7 +249,9 @@ pub fn init() -> KResult<()> { let serial = Serial::new(0, port)?; serial.register_as_char_device( - // 2 or 4 here, let's try 2 first! + // UART0 is wired to PCH-PIC line 2 on the QEMU virt machine + // (`interrupts = <2 4>` in the device tree). The HAL routes it + // to an EIOINTC vector and delivers it as an external IRQ. 2, )?; } diff --git a/src/driver/serial/io.rs b/src/driver/serial/io.rs index aec18f20..57e61c56 100644 --- a/src/driver/serial/io.rs +++ b/src/driver/serial/io.rs @@ -1,10 +1,11 @@ -use super::SerialRegister; use core::ptr::NonNull; -use eonix_hal::{fence::memory_barrier, mm::ArchPhysAccess}; -use eonix_mm::address::{PAddr, PhysAccess}; #[cfg(target_arch = "x86_64")] use eonix_hal::arch_exported::io::Port8; +use eonix_hal::mm::ArchPhysAccess; +use eonix_mm::address::{PAddr, PhysAccess}; + +use super::SerialRegister; #[cfg(target_arch = "x86_64")] pub struct SerialIO { @@ -73,10 +74,12 @@ impl SerialIO { self.line_status } + #[allow(unused)] pub fn modem_status(&self) -> impl SerialRegister { self.modem_status } + #[allow(unused)] pub fn scratch(&self) -> impl SerialRegister { self.scratch } @@ -100,7 +103,7 @@ impl SerialRegister for NonNull<u8> { let retval = unsafe { self.as_ptr().read_volatile() }; #[cfg(target_arch = "loongarch64")] - memory_barrier(); + eonix_hal::fence::memory_barrier(); retval } @@ -110,7 +113,7 @@ impl SerialRegister for NonNull<u8> { unsafe { self.as_ptr().write_volatile(data) }; #[cfg(target_arch = "loongarch64")] - memory_barrier(); + eonix_hal::fence::memory_barrier(); } } @@ -155,10 +158,12 @@ impl SerialIO { unsafe { self.base_addr.add(5) } } + #[allow(unused)] pub fn modem_status(&self) -> impl SerialRegister { unsafe { self.base_addr.add(6) } } + #[allow(unused)] pub fn scratch(&self) -> impl SerialRegister { unsafe { self.base_addr.add(7) } } diff --git a/src/driver/virtio/riscv64.rs b/src/driver/virtio/riscv64.rs index 9bdbf6ce..b33e16ac 100644 --- a/src/driver/virtio/riscv64.rs +++ b/src/driver/virtio/riscv64.rs @@ -1,8 +1,5 @@ use super::virtio_blk::HAL; -use crate::kernel::{ - block::{make_device, BlockDevice}, - task::block_on, -}; +use crate::kernel::{block::BlockDevice, task::block_on, vfs::types::DeviceId}; use alloc::{sync::Arc, vec::Vec}; use eonix_hal::arch_exported::fdt::FDT; use eonix_hal::mm::ArchPhysAccess; @@ -43,7 +40,7 @@ pub fn init() { .expect("Failed to initialize VirtIO Block device"); let block_device = BlockDevice::register_disk( - make_device(8, 16 * disk_id), + DeviceId::new(8, 16 * disk_id), 2147483647, Arc::new(Spin::new(block_device)), ) diff --git a/src/driver/virtio/virtio_blk.rs b/src/driver/virtio/virtio_blk.rs index 37e4fe77..5dfed88a 100644 --- a/src/driver/virtio/virtio_blk.rs +++ b/src/driver/virtio/virtio_blk.rs @@ -1,19 +1,19 @@ -use crate::{ - io::Chunks, - kernel::{ - block::{BlockDeviceRequest, BlockRequestQueue}, - constants::EIO, - mem::{AsMemoryBlock, Page}, - }, - prelude::KResult, -}; +use alloc::boxed::Box; + +use async_trait::async_trait; use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::{ - address::{Addr, PAddr, PhysAccess}, - paging::PFN, -}; +use eonix_mm::address::{Addr, PAddr, PhysAccess}; +use eonix_mm::paging::{Folio as _, PFN}; use eonix_sync::Spin; -use virtio_drivers::{device::blk::VirtIOBlk, transport::Transport, Hal}; +use virtio_drivers::device::blk::VirtIOBlk; +use virtio_drivers::transport::Transport; +use virtio_drivers::Hal; + +use crate::io::Chunks; +use crate::kernel::block::{BlockDeviceRequest, BlockRequestQueue}; +use crate::kernel::constants::EIO; +use crate::kernel::mem::Folio; +use crate::prelude::KResult; pub struct HAL; @@ -22,13 +22,12 @@ unsafe impl Hal for HAL { pages: usize, _direction: virtio_drivers::BufferDirection, ) -> (virtio_drivers::PhysAddr, core::ptr::NonNull<u8>) { - let page = Page::alloc_at_least(pages); + let page = Folio::alloc_at_least(pages); - let paddr = page.start().addr(); - let ptr = page.as_memblk().as_byte_ptr(); - page.into_raw(); + let ptr = page.get_ptr(); + let pfn = page.into_raw(); - (paddr, ptr) + (PAddr::from(pfn).addr(), ptr) } unsafe fn dma_dealloc( @@ -41,7 +40,7 @@ unsafe impl Hal for HAL { unsafe { // SAFETY: The caller ensures that the pfn corresponds to a valid // page allocated by `dma_alloc`. - Page::from_raw(pfn); + Folio::from_raw(pfn); } 0 @@ -74,6 +73,7 @@ unsafe impl Hal for HAL { } } +#[async_trait] impl<T> BlockRequestQueue for Spin<VirtIOBlk<HAL, T>> where T: Transport + Send, @@ -82,7 +82,7 @@ where 1024 } - fn submit(&self, req: BlockDeviceRequest) -> KResult<()> { + async fn submit<'a>(&'a self, req: BlockDeviceRequest<'a>) -> KResult<()> { match req { BlockDeviceRequest::Write { sector, @@ -90,15 +90,14 @@ where buffer, } => { let mut dev = self.lock(); - for ((start, len), buffer_page) in + for ((start, sectors), buffer_page) in Chunks::new(sector as usize, count as usize, 8).zip(buffer.iter()) { - let buffer = unsafe { - // SAFETY: Pages in `req.buffer` are guaranteed to be exclusively owned by us. - &buffer_page.as_memblk().as_bytes()[..len as usize * 512] - }; + let len = sectors * 512; + let pg = buffer_page.lock(); - dev.write_blocks(start, buffer).map_err(|_| EIO)?; + dev.write_blocks(start, &pg.as_bytes()[..len]) + .map_err(|_| EIO)?; } } BlockDeviceRequest::Read { @@ -107,15 +106,14 @@ where buffer, } => { let mut dev = self.lock(); - for ((start, len), buffer_page) in + for ((start, sectors), buffer_page) in Chunks::new(sector as usize, count as usize, 8).zip(buffer.iter()) { - let buffer = unsafe { - // SAFETY: Pages in `req.buffer` are guaranteed to be exclusively owned by us. - &mut buffer_page.as_memblk().as_bytes_mut()[..len as usize * 512] - }; + let len = sectors * 512; + let mut pg = buffer_page.lock(); - dev.read_blocks(start, buffer).map_err(|_| EIO)?; + dev.read_blocks(start, &mut pg.as_bytes_mut()[..len]) + .map_err(|_| EIO)?; } } } diff --git a/src/fs/ext4.rs b/src/fs/ext4.rs index 76ca4a34..121339d3 100644 --- a/src/fs/ext4.rs +++ b/src/fs/ext4.rs @@ -1,5 +1,3 @@ -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; - use crate::kernel::mem::{CachePage, CachePageStream, PageCache, PageCacheBackend}; use crate::kernel::task::block_on; use crate::kernel::timer::Ticks; @@ -31,6 +29,8 @@ use alloc::{ use another_ext4::{ Block, BlockDevice as Ext4BlockDeviceTrait, Ext4, FileType, InodeMode, PBlockId, }; +use async_trait::async_trait; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use eonix_sync::RwLock; pub struct Ext4BlockDevice { @@ -194,7 +194,7 @@ impl Ext4Fs { root_inode.inode.mtime_extra() as _, )), rwsem: RwLock::new(()), - vfs: Arc::downgrade(&ext4fs) as _, + sb: Arc::downgrade(&ext4fs) as _, }, ) }; @@ -290,7 +290,7 @@ impl Inode for FileInode { } fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let mut temp_buf = vec![0u8; buffer.total()]; @@ -334,7 +334,7 @@ impl Inode for FileInode { fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult<usize> { //let _lock = Task::block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let mut temp_buf = vec![0u8; 4096]; @@ -363,7 +363,7 @@ impl Inode for FileInode { fn chmod(&self, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let old_mode = self.mode.load(); let new_mode = old_mode.perm(mode.bits()); @@ -428,7 +428,7 @@ impl DirInode { impl Inode for DirInode { fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<Arc<dyn Inode>>> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let name = dentry.get_name(); @@ -477,7 +477,7 @@ impl Inode for DirInode { ctime: Spin::new(Instant::new(attr.ctime as _, 0)), mtime: Spin::new(Instant::new(attr.mtime as _, 0)), rwsem: RwLock::new(()), - vfs: self.vfs.clone(), + sb: self.sb.clone(), }, ); @@ -489,7 +489,7 @@ impl Inode for DirInode { offset: usize, callback: &mut dyn FnMut(&[u8], Ino) -> KResult<core::ops::ControlFlow<(), ()>>, ) -> KResult<usize> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let entries = ext4fs @@ -519,7 +519,7 @@ impl Inode for DirInode { fn creat(&self, at: &Arc<Dentry>, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let name = at.get_name(); @@ -534,7 +534,7 @@ impl Inode for DirInode { ) .unwrap(); - let file = FileInode::new(new_ino as u64, self.vfs.clone(), mode); + let file = FileInode::new(new_ino as u64, self.sb.clone(), mode); let now = Instant::now(); self.update_child_time(file.as_ref(), now); self.link_file(); @@ -547,7 +547,7 @@ impl Inode for DirInode { fn mkdir(&self, at: &Dentry, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let name = at.get_name(); @@ -562,7 +562,7 @@ impl Inode for DirInode { ) .unwrap(); - let new_dir = DirInode::new(new_ino as u64, self.vfs.clone(), mode); + let new_dir = DirInode::new(new_ino as u64, self.sb.clone(), mode); let now = Instant::now(); self.update_child_time(new_dir.as_ref(), now); self.link_dir(); @@ -575,7 +575,7 @@ impl Inode for DirInode { fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { let _dir_lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let file = at.get_inode()?; @@ -602,7 +602,7 @@ impl Inode for DirInode { fn chmod(&self, mode: Mode) -> KResult<()> { let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let old_mode = self.mode.load(); let new_mode = old_mode.perm(mode.bits()); @@ -638,7 +638,7 @@ impl Inode for DirInode { // TODO: may need another lock let _lock = block_on(self.rwsem.write()); - let vfs = self.vfs.upgrade().ok_or(EIO)?; + let vfs = self.sb.upgrade().ok_or(EIO)?; let ext4fs = vfs.as_any().downcast_ref::<Ext4Fs>().unwrap(); let old_file = old_dentry.get_inode()?; @@ -698,6 +698,7 @@ impl From<InodeMode> for Mode { struct Ext4MountCreator; +#[async_trait] impl MountCreator for Ext4MountCreator { fn check_signature(&self, mut first_block: &[u8]) -> KResult<bool> { match first_block.split_off(1080..) { @@ -707,7 +708,7 @@ impl MountCreator for Ext4MountCreator { } } - fn create_mount(&self, source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { + async fn create_mount(&self, source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { let source = source.as_bytes(); let path = Path::new(source)?; diff --git a/src/fs/fat32.rs b/src/fs/fat32.rs index 12eabdd5..c1feebdf 100644 --- a/src/fs/fat32.rs +++ b/src/fs/fat32.rs @@ -1,45 +1,68 @@ mod dir; -mod file; - -use crate::io::Stream; -use crate::kernel::constants::EIO; -use crate::kernel::mem::{AsMemoryBlock, CachePageStream}; -use crate::kernel::task::block_on; -use crate::kernel::vfs::inode::{Mode, WriteOffset}; -use crate::{ - io::{Buffer, ByteBuffer, UninitBuffer}, - kernel::{ - block::{make_device, BlockDevice, BlockDeviceRequest}, - mem::{ - paging::Page, - {CachePage, PageCache, PageCacheBackend}, - }, - vfs::{ - dentry::Dentry, - inode::{define_struct_inode, Ino, Inode, InodeData}, - mount::{register_filesystem, Mount, MountCreator}, - vfs::Vfs, - DevId, - }, - }, - prelude::*, - KResult, -}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, - vec::Vec, -}; -use core::{ops::ControlFlow, sync::atomic::Ordering}; -use dir::Dirs as _; + +use alloc::sync::Arc; +use core::ops::Deref; + +use async_trait::async_trait; +use dir::{as_raw_dirents, ParseDirent}; +use eonix_mm::paging::PAGE_SIZE; use eonix_sync::RwLock; -use file::ClusterRead; +use itertools::Itertools; + +use crate::io::{Buffer, ByteBuffer, UninitBuffer}; +use crate::kernel::block::{BlockDevice, BlockDeviceRequest}; +use crate::kernel::constants::{EINVAL, EIO}; +use crate::kernel::mem::{CachePage, Folio, FolioOwned, PageOffset}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; +use crate::kernel::vfs::types::{DeviceId, Format, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; +use crate::prelude::*; +use crate::KResult; + +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Cluster(u32); + +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +struct RawCluster(pub u32); + +impl RawCluster { + const START: u32 = 2; + const EOC: u32 = 0x0FFF_FFF8; + const INVL: u32 = 0xF000_0000; + + fn parse(self) -> Option<Cluster> { + match self.0 { + ..Self::START | Self::EOC..Self::INVL => None, + Self::INVL.. => { + unreachable!("invalid cluster number: RawCluster({:#08x})", self.0) + } + no => Some(Cluster(no)), + } + } +} -type ClusterNo = u32; +impl Cluster { + pub fn as_ino(self) -> Ino { + Ino::new(self.0 as _) + } + + pub fn from_ino(ino: Ino) -> Self { + Self(ino.as_raw() as u32) + } + + fn normalized(self) -> Self { + Self(self.0 - 2) + } +} const SECTOR_SIZE: usize = 512; -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug)] #[repr(C, packed)] struct Bootsector { jmp: [u8; 3], @@ -59,7 +82,7 @@ struct Bootsector { sectors_per_fat: u32, flags: u16, fat_version: u16, - root_cluster: ClusterNo, + root_cluster: RawCluster, fsinfo_sector: u16, backup_bootsector: u16, _reserved: [u8; 12], @@ -73,341 +96,355 @@ struct Bootsector { mbr_signature: u16, } -impl_any!(FatFs); /// # Lock order /// 2. FatTable /// 3. Inodes /// struct FatFs { sectors_per_cluster: u8, - rootdir_cluster: ClusterNo, - data_start: u64, - volume_label: [u8; 11], + data_start_sector: u64, + _rootdir_cluster: Cluster, + _volume_label: Box<str>, device: Arc<BlockDevice>, - fat: RwLock<Vec<ClusterNo>>, - weak: Weak<FatFs>, - icache: BTreeMap<Ino, FatInode>, + fat: RwLock<Box<[RawCluster]>>, } -impl Vfs for FatFs { - fn io_blksize(&self) -> usize { - 4096 - } - - fn fs_devid(&self) -> DevId { - self.device.devid() - } - - fn is_read_only(&self) -> bool { - true - } -} +impl SuperBlock for FatFs {} impl FatFs { - fn read_cluster(&self, cluster: ClusterNo, buf: &Page) -> KResult<()> { - let cluster = cluster - 2; + async fn read_cluster(&self, mut cluster: Cluster, buf: &Folio) -> KResult<()> { + cluster = cluster.normalized(); let rq = BlockDeviceRequest::Read { - sector: self.data_start as u64 + cluster as u64 * self.sectors_per_cluster as u64, + sector: self.data_start_sector as u64 + + cluster.0 as u64 * self.sectors_per_cluster as u64, count: self.sectors_per_cluster as u64, buffer: core::slice::from_ref(buf), }; - self.device.commit_request(rq)?; + self.device.commit_request(rq).await?; Ok(()) } - - fn get_or_alloc_inode(&self, ino: Ino, is_directory: bool, size: u32) -> Arc<dyn Inode> { - self.icache - .get(&ino) - .cloned() - .map(FatInode::unwrap) - .unwrap_or_else(|| { - if is_directory { - DirInode::new(ino, self.weak.clone(), size) - } else { - FileInode::new(ino, self.weak.clone(), size) - } - }) - } } impl FatFs { - pub fn create(device: DevId) -> KResult<(Arc<Self>, Arc<dyn Inode>)> { + pub async fn create(device: DeviceId) -> KResult<(SbUse<Self>, InodeUse)> { let device = BlockDevice::get(device)?; - let mut fatfs_arc = Arc::new_cyclic(|weak: &Weak<FatFs>| Self { - device, - sectors_per_cluster: 0, - rootdir_cluster: 0, - data_start: 0, - fat: RwLock::new(Vec::new()), - weak: weak.clone(), - icache: BTreeMap::new(), - volume_label: [0; 11], - }); - - let fatfs = unsafe { Arc::get_mut_unchecked(&mut fatfs_arc) }; - - let mut info: UninitBuffer<Bootsector> = UninitBuffer::new(); - fatfs.device.read_some(0, &mut info)?.ok_or(EIO)?; + + let mut info = UninitBuffer::<Bootsector>::new(); + device.read_some(0, &mut info).await?.ok_or(EIO)?; let info = info.assume_filled_ref()?; - fatfs.sectors_per_cluster = info.sectors_per_cluster; - fatfs.rootdir_cluster = info.root_cluster; - fatfs.data_start = - info.reserved_sectors as u64 + info.fat_copies as u64 * info.sectors_per_fat as u64; + let mut fat = Box::new_uninit_slice( + 512 * info.sectors_per_fat as usize / core::mem::size_of::<Cluster>(), + ); - let fat = fatfs.fat.get_mut(); + device + .read_some( + info.reserved_sectors as usize * 512, + &mut ByteBuffer::from(fat.as_mut()), + ) + .await? + .ok_or(EIO)?; - fat.resize( - 512 * info.sectors_per_fat as usize / core::mem::size_of::<ClusterNo>(), - 0, - ); + let sectors_per_cluster = info.sectors_per_cluster; + let rootdir_cluster = info.root_cluster.parse().ok_or(EINVAL)?; - let mut buffer = ByteBuffer::from(fat.as_mut_slice()); + let data_start_sector = + info.reserved_sectors as u64 + info.fat_copies as u64 * info.sectors_per_fat as u64; - fatfs - .device - .read_some(info.reserved_sectors as usize * 512, &mut buffer)? - .ok_or(EIO)?; + let volume_label = { + let end = info + .volume_label + .iter() + .position(|&c| c == b' ') + .unwrap_or(info.volume_label.len()); - info.volume_label - .iter() - .take_while(|&&c| c != ' ' as u8) - .take(11) - .enumerate() - .for_each(|(idx, c)| fatfs.volume_label[idx] = *c); - - let root_dir_cluster_count = ClusterIterator::new(fat, fatfs.rootdir_cluster).count(); - let root_dir_size = root_dir_cluster_count as u32 * info.sectors_per_cluster as u32 * 512; - - let root_inode = DirInode::new( - (info.root_cluster & !0xF000_0000) as Ino, - fatfs.weak.clone(), - root_dir_size, + String::from_utf8_lossy(&info.volume_label[..end]) + .into_owned() + .into_boxed_str() + }; + + let fat = unsafe { fat.assume_init() }; + + let rootdir_cluster_count = ClusterIterator::new(fat.as_ref(), rootdir_cluster).count(); + let rootdir_size = rootdir_cluster_count as u32 * sectors_per_cluster as u32 * 512; + + let fatfs = SbUse::new( + SuperBlockInfo { + io_blksize: 4096, + device_id: device.devid(), + read_only: true, + }, + Self { + device, + sectors_per_cluster, + _rootdir_cluster: rootdir_cluster, + data_start_sector, + fat: RwLock::new(fat), + _volume_label: volume_label, + }, ); - Ok((fatfs_arc, root_inode)) + let sbref = SbRef::from(&fatfs); + Ok((fatfs, DirInode::new(rootdir_cluster, sbref, rootdir_size))) } } -struct ClusterIterator<'fat> { - fat: &'fat [ClusterNo], - cur: ClusterNo, +struct ClusterIterator<'a> { + fat: &'a [RawCluster], + cur: Option<Cluster>, } -impl<'fat> ClusterIterator<'fat> { - fn new(fat: &'fat [ClusterNo], start: ClusterNo) -> Self { - Self { fat, cur: start } +impl<'a> ClusterIterator<'a> { + fn new(fat: &'a [RawCluster], start: Cluster) -> Self { + Self { + fat, + cur: Some(start), + } } } impl<'fat> Iterator for ClusterIterator<'fat> { - type Item = ClusterNo; + type Item = Cluster; fn next(&mut self) -> Option<Self::Item> { - const EOC: ClusterNo = 0x0FFF_FFF8; - const INVL: ClusterNo = 0xF000_0000; - - match self.cur { - ..2 | EOC..INVL => None, - INVL.. => unreachable!("Invalid cluster number: {}", self.cur), - next => { - self.cur = self.fat[next as usize] & !INVL; - Some(next) - } - } + self.cur.inspect(|&Cluster(no)| { + self.cur = self.fat[no as usize].parse(); + }) } } -#[allow(dead_code)] -#[derive(Clone)] -enum FatInode { - File(Arc<FileInode>), - Dir(Arc<DirInode>), -} +struct FileInode; -impl FatInode { - fn unwrap(self) -> Arc<dyn Inode> { - match self { - FatInode::File(inode) => inode, - FatInode::Dir(inode) => inode, - } +impl FileInode { + fn new(cluster: Cluster, sb: SbRef<FatFs>, size: u32) -> InodeUse { + InodeUse::new( + sb, + cluster.as_ino(), + Format::REG, + InodeInfo { + size: size as u64, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o777), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }, + Self, + ) } } -define_struct_inode! { - struct FileInode { - page_cache: PageCache, +impl InodeOps for FileInode { + type SuperBlock = FatFs; + + async fn read( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + buffer: &mut dyn Buffer, + offset: usize, + ) -> KResult<usize> { + inode.get_page_cache().read(buffer, offset).await } -} -impl FileInode { - fn new(ino: Ino, weak: Weak<FatFs>, size: u32) -> Arc<Self> { - let inode = Arc::new_cyclic(|weak_self: &Weak<FileInode>| Self { - idata: InodeData::new(ino, weak), - page_cache: PageCache::new(weak_self.clone()), - }); - - // Safety: We are initializing the inode - inode.nlink.store(1, Ordering::Relaxed); - inode.mode.store(Mode::REG.perm(0o777)); - inode.size.store(size as u64, Ordering::Relaxed); - - inode + async fn read_page( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + page: &mut CachePage, + offset: PageOffset, + ) -> KResult<()> { + let fs = &sb.backend; + let fat = sb.backend.fat.read().await; + + if offset >= PageOffset::from_byte_ceil(inode.info.lock().size as usize) { + unreachable!("read_page called with offset beyond file size"); + } + + let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; + if cluster_size != PAGE_SIZE { + unimplemented!("cluster size != PAGE_SIZE"); + } + + // XXX: Ugly and inefficient O(n^2) algorithm for sequential file read. + let cluster = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)) + .skip(offset.page_count()) + .next() + .ok_or(EIO)?; + + fs.read_cluster(cluster, &page).await?; + + let real_len = (inode.info.lock().size as usize) - offset.byte_count(); + if real_len < PAGE_SIZE { + let mut page = page.lock(); + page.as_bytes_mut()[real_len..].fill(0); + } + + Ok(()) } } -impl Inode for FileInode { - fn page_cache(&self) -> Option<&PageCache> { - Some(&self.page_cache) - } +struct DirInode { + // TODO: Use the new PageCache... + dir_pages: RwLock<Vec<FolioOwned>>, +} - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - block_on(self.page_cache.read(buffer, offset)) +impl DirInode { + fn new(cluster: Cluster, sb: SbRef<FatFs>, size: u32) -> InodeUse { + InodeUse::new( + sb, + cluster.as_ino(), + Format::DIR, + InodeInfo { + size: size as u64, + nlink: 2, // '.' and '..' + uid: 0, + gid: 0, + perm: Permission::new(0o777), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }, + Self { + dir_pages: RwLock::new(Vec::new()), + }, + ) } - fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; - let vfs = vfs.as_any().downcast_ref::<FatFs>().unwrap(); - let fat = block_on(vfs.fat.read()); - - if self.size.load(Ordering::Relaxed) as usize == 0 { - return Ok(0); + async fn read_dir_pages(&self, sb: &SbUse<FatFs>, inode: &InodeUse) -> KResult<()> { + let mut dir_pages = self.dir_pages.write().await; + if !dir_pages.is_empty() { + return Ok(()); } - let cluster_size = vfs.sectors_per_cluster as usize * SECTOR_SIZE; - assert!(cluster_size <= 0x1000, "Cluster size is too large"); + let fs = &sb.backend; + let fat = fs.fat.read().await; - let skip_clusters = offset / cluster_size; - let inner_offset = offset % cluster_size; - - let cluster_iter = - ClusterIterator::new(fat.as_ref(), self.ino as ClusterNo).skip(skip_clusters); + let clusters = ClusterIterator::new(fat.as_ref(), Cluster::from_ino(inode.ino)); - let buffer_page = Page::alloc(); - for cluster in cluster_iter { - vfs.read_cluster(cluster, &buffer_page)?; + for cluster in clusters { + let page = FolioOwned::alloc(); + fs.read_cluster(cluster, &page).await?; - let data = unsafe { - // SAFETY: We are the only one holding this page. - &buffer_page.as_memblk().as_bytes()[inner_offset..] - }; + dir_pages.push(page); + } - let end = offset + data.len(); - let real_end = core::cmp::min(end, self.size.load(Ordering::Relaxed) as usize); - let real_size = real_end - offset; + Ok(()) + } - if buffer.fill(&data[..real_size])?.should_stop() { - break; + async fn get_dir_pages( + &self, + sb: &SbUse<FatFs>, + inode: &InodeUse, + ) -> KResult<impl Deref<Target = Vec<FolioOwned>> + use<'_>> { + { + let dir_pages = self.dir_pages.read().await; + if !dir_pages.is_empty() { + return Ok(dir_pages); } } - Ok(buffer.wrote()) - } + self.read_dir_pages(sb, inode).await?; - fn write(&self, _stream: &mut dyn Stream, _offset: WriteOffset) -> KResult<usize> { - todo!() - } + if let Some(dir_pages) = self.dir_pages.try_read() { + return Ok(dir_pages); + } - fn write_direct(&self, _stream: &mut dyn Stream, _offset: usize) -> KResult<usize> { - todo!() + Ok(self.dir_pages.read().await) } } -impl PageCacheBackend for FileInode { - fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize> { - self.read_direct(page, offset) - } +impl InodeOps for DirInode { + type SuperBlock = FatFs; - fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult<usize> { - todo!() - } + async fn lookup( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + dentry: &Arc<Dentry>, + ) -> KResult<Option<InodeUse>> { + let dir_pages = self.get_dir_pages(&sb, inode).await?; - fn size(&self) -> usize { - self.size.load(Ordering::Relaxed) as usize - } -} + let dir_data = dir_pages.iter().map(|pg| pg.as_bytes()); -define_struct_inode! { - struct DirInode; -} + let raw_dirents = dir_data + .map(as_raw_dirents) + .take_while_inclusive(Result::is_ok) + .flatten_ok(); -impl DirInode { - fn new(ino: Ino, weak: Weak<FatFs>, size: u32) -> Arc<Self> { - let inode = Arc::new(Self { - idata: InodeData::new(ino, weak), - }); + let mut dirents = futures::stream::iter(raw_dirents); - // Safety: We are initializing the inode - inode.nlink.store(2, Ordering::Relaxed); - inode.mode.store(Mode::DIR.perm(0o777)); - inode.size.store(size as u64, Ordering::Relaxed); + while let Some(result) = dirents.next_dirent().await { + let entry = result?; - inode - } -} + if *entry.filename != ****dentry.name() { + continue; + } -impl Inode for DirInode { - fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<Arc<dyn Inode>>> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; - let vfs = vfs.as_any().downcast_ref::<FatFs>().unwrap(); - let fat = block_on(vfs.fat.read()); - - let mut entries = ClusterIterator::new(fat.as_ref(), self.ino as ClusterNo) - .read(vfs, 0) - .dirs(); - - let entry = entries.find(|entry| { - entry - .as_ref() - .map(|entry| &entry.filename == &***dentry.name()) - .unwrap_or(true) - }); - - match entry { - None => Ok(None), - Some(Err(err)) => Err(err), - Some(Ok(entry)) => Ok(Some(vfs.get_or_alloc_inode( - entry.cluster as Ino, - entry.is_directory, - entry.size, - ))), + let sbref = SbRef::from(&sb); + + if entry.is_directory { + return Ok(Some(DirInode::new(entry.cluster, sbref, entry.size) as _)); + } else { + return Ok(Some(FileInode::new(entry.cluster, sbref, entry.size) as _)); + } } + + Ok(None) } - fn do_readdir( + async fn readdir( &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, - ) -> KResult<usize> { - let vfs = self.vfs.upgrade().ok_or(EIO)?; - let vfs = vfs.as_any().downcast_ref::<FatFs>().unwrap(); - let fat = block_on(vfs.fat.read()); + callback: &mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> KResult<KResult<usize>> { + let fs = &sb.backend; + let dir_pages = self.get_dir_pages(&sb, inode).await?; - let cluster_iter = ClusterIterator::new(fat.as_ref(), self.ino as ClusterNo) - .read(vfs, offset) - .dirs(); + let cluster_size = fs.sectors_per_cluster as usize * SECTOR_SIZE; - let mut nread = 0usize; - for entry in cluster_iter { - let entry = entry?; + let cluster_offset = offset / cluster_size; + let inner_offset = offset % cluster_size; + let inner_raw_dirent_offset = inner_offset / core::mem::size_of::<dir::RawDirEntry>(); - vfs.get_or_alloc_inode(entry.cluster as Ino, entry.is_directory, entry.size); - if callback(&entry.filename, entry.cluster as Ino)?.is_break() { - break; - } + let dir_data = dir_pages + .iter() + .skip(cluster_offset) + .map(|pg| pg.as_bytes()); + + let raw_dirents = dir_data + .map(as_raw_dirents) + .take_while_inclusive(Result::is_ok) + .flatten_ok() + .skip(inner_raw_dirent_offset); + + let mut dirents = futures::stream::iter(raw_dirents); - nread += entry.entry_offset as usize; + let mut nread = 0; + while let Some(result) = dirents.next_dirent().await { + let entry = result?; + + match callback(&entry.filename, entry.cluster.as_ino()) { + Err(err) => return Ok(Err(err)), + Ok(true) => nread += entry.entry_offset as usize, + Ok(false) => break, + } } - Ok(nread) + Ok(Ok(nread)) } } struct FatMountCreator; +#[async_trait] impl MountCreator for FatMountCreator { fn check_signature(&self, mut first_block: &[u8]) -> KResult<bool> { match first_block.split_off(82..) { @@ -417,8 +454,8 @@ impl MountCreator for FatMountCreator { } } - fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { - let (fatfs, root_inode) = FatFs::create(make_device(8, 1))?; + async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { + let (fatfs, root_inode) = FatFs::create(DeviceId::new(8, 1)).await?; Mount::new(mp, fatfs, root_inode) } diff --git a/src/fs/fat32/dir.rs b/src/fs/fat32/dir.rs index c4ac6c0d..8a5b6f40 100644 --- a/src/fs/fat32/dir.rs +++ b/src/fs/fat32/dir.rs @@ -1,11 +1,16 @@ -use super::file::ClusterReadIterator; +use core::pin::Pin; + +use alloc::{boxed::Box, string::String}; +use futures::{Stream, StreamExt}; +use posix_types::result::PosixError; + use crate::kernel::constants::EINVAL; use crate::prelude::*; -use alloc::{string::String, sync::Arc}; -use itertools::Itertools; + +use super::{Cluster, RawCluster}; #[repr(C, packed)] -pub(super) struct RawDirEntry { +pub struct RawDirEntry { name: [u8; 8], extension: [u8; 3], attr: u8, @@ -21,9 +26,9 @@ pub(super) struct RawDirEntry { size: u32, } -pub(super) struct FatDirectoryEntry { - pub filename: Arc<[u8]>, - pub cluster: u32, +pub struct FatDirectoryEntry { + pub filename: Box<[u8]>, + pub cluster: Cluster, pub size: u32, pub entry_offset: u32, pub is_directory: bool, @@ -79,7 +84,7 @@ impl RawDirEntry { self.attr & Self::ATTR_DIRECTORY != 0 } - fn long_filename(&self) -> Option<[u16; 13]> { + fn as_raw_long_filename(&self) -> Option<[u16; 13]> { if !self.is_long_filename() { return None; } @@ -103,137 +108,114 @@ impl RawDirEntry { } } -impl<'data, I> RawDirs<'data> for I where I: ClusterReadIterator<'data> {} -trait RawDirs<'data>: ClusterReadIterator<'data> { - fn raw_dirs(self) -> impl Iterator<Item = KResult<&'data RawDirEntry>> + 'data - where - Self: Sized, - { - const ENTRY_SIZE: usize = size_of::<RawDirEntry>(); - - self.map(|result| { - let data = result?; - if data.len() % ENTRY_SIZE != 0 { - return Err(EINVAL); - } - - Ok(unsafe { - core::slice::from_raw_parts( - data.as_ptr() as *const RawDirEntry, - data.len() / ENTRY_SIZE, - ) - }) - }) - .flatten_ok() +pub fn as_raw_dirents(data: &[u8]) -> KResult<&[RawDirEntry]> { + let len = data.len(); + if len % size_of::<RawDirEntry>() != 0 { + return Err(EINVAL); } -} - -pub(super) trait Dirs<'data>: ClusterReadIterator<'data> { - fn dirs(self) -> impl Iterator<Item = KResult<FatDirectoryEntry>> + 'data - where - Self: Sized; -} -impl<'data, I> Dirs<'data> for I -where - I: ClusterReadIterator<'data>, -{ - fn dirs(self) -> impl Iterator<Item = KResult<FatDirectoryEntry>> + 'data - where - Self: Sized, - { - self.raw_dirs().real_dirs() + unsafe { + Ok(core::slice::from_raw_parts( + data.as_ptr() as *const RawDirEntry, + len / size_of::<RawDirEntry>(), + )) } } -trait RealDirs<'data>: Iterator<Item = KResult<&'data RawDirEntry>> + 'data { - fn real_dirs(self) -> DirsIter<'data, Self> - where - Self: Sized; +pub trait ParseDirent { + async fn next_dirent(&mut self) -> Option<KResult<FatDirectoryEntry>>; } -impl<'data, I> RealDirs<'data> for I +impl<'a, T> ParseDirent for T where - I: Iterator<Item = KResult<&'data RawDirEntry>> + 'data, + T: Stream<Item = KResult<&'a RawDirEntry>>, { - fn real_dirs(self) -> DirsIter<'data, Self> - where - Self: Sized, - { - DirsIter { iter: self } - } -} + async fn next_dirent(&mut self) -> Option<KResult<FatDirectoryEntry>> { + let mut me = unsafe { Pin::new_unchecked(self) }; + + // The long filename entries are stored in reverse order. + // So we reverse all filename segments and then reverse the whole string at the end. + let mut filename_rev = String::new(); + + let mut is_lfn = false; + let mut nr_entry_scanned = 0; + let mut cur_entry; + + loop { + match me.as_mut().next().await { + Some(Err(err)) => return Some(Err(err)), + Some(Ok(ent)) => { + cur_entry = ent; + nr_entry_scanned += 1; + } + None => { + if is_lfn { + // Unterminated long filename entries are invalid. + return Some(Err(PosixError::EINVAL.into())); + } else { + return None; + } + } + }; -pub(super) struct DirsIter<'data, I> -where - I: Iterator<Item = KResult<&'data RawDirEntry>> + 'data, -{ - iter: I, -} + if !cur_entry.is_invalid() { + break; + } -impl<'data, I> Iterator for DirsIter<'data, I> -where - I: Iterator<Item = KResult<&'data RawDirEntry>> + 'data, -{ - type Item = KResult<FatDirectoryEntry>; - - fn next(&mut self) -> Option<Self::Item> { - let mut filename = String::new(); - let mut entry_offset = 0; - let entry = loop { - let entry = match self.iter.next()? { - Ok(entry) => entry, - Err(err) => return Some(Err(err)), - }; - entry_offset += 1; - - let long_filename = entry.long_filename(); - if entry.is_invalid() { - if let Some(long_filename) = long_filename { - let long_filename = long_filename - .iter() - .position(|&ch| ch == 0) - .map(|pos| &long_filename[..pos]) - .unwrap_or(&long_filename); - - filename.extend( - long_filename - .into_iter() - .map(|&ch| char::from_u32(ch as u32).unwrap_or('?')) - .rev(), - ); - } + let Some(raw_long_filename) = cur_entry.as_raw_long_filename() else { continue; - } - break entry; + }; + + // We are processing a long filename entry. + is_lfn = true; + + let real_len = raw_long_filename + .iter() + .position(|&ch| ch == 0) + .unwrap_or(raw_long_filename.len()); + + let name_codes_rev = raw_long_filename.into_iter().take(real_len).rev(); + let name_chars_rev = char::decode_utf16(name_codes_rev).map(|r| r.unwrap_or('?')); + + filename_rev.extend(name_chars_rev); + } + + // From now on, `entry` represents a valid directory entry. + + let raw_cluster = + RawCluster(cur_entry.cluster_low as u32 | ((cur_entry.cluster_high as u32) << 16)); + + let Some(cluster) = raw_cluster.parse() else { + return Some(Err(PosixError::EINVAL.into())); }; - let filename: Arc<[u8]> = if filename.is_empty() { - let mut filename = entry.filename().to_vec(); - let extension = entry.extension(); + let filename; + + if filename_rev.is_empty() { + let mut name = cur_entry.filename().to_vec(); + let extension = cur_entry.extension(); if !extension.is_empty() { - filename.push(b'.'); - filename.extend_from_slice(extension); + name.push(b'.'); + name.extend_from_slice(extension); } - if entry.is_filename_lowercase() { - filename.make_ascii_lowercase(); + if cur_entry.is_filename_lowercase() { + name.make_ascii_lowercase(); } - filename.into() + filename = name.into_boxed_slice(); } else { - let mut bytes = filename.into_bytes(); - bytes.reverse(); - - bytes.into() - }; + let mut name = filename_rev.into_bytes(); + name.reverse(); + filename = name.into_boxed_slice(); + } Some(Ok(FatDirectoryEntry { - size: entry.size, - entry_offset, + size: cur_entry.size, + entry_offset: nr_entry_scanned * size_of::<RawDirEntry>() as u32, filename, - cluster: entry.cluster_low as u32 | (((entry.cluster_high & !0xF000) as u32) << 16), - is_directory: entry.is_directory(), + cluster, + is_directory: cur_entry.is_directory(), })) } } diff --git a/src/fs/fat32/file.rs b/src/fs/fat32/file.rs deleted file mode 100644 index db16df50..00000000 --- a/src/fs/fat32/file.rs +++ /dev/null @@ -1,40 +0,0 @@ -use super::{ClusterIterator, FatFs}; -use crate::{ - kernel::mem::{AsMemoryBlock as _, Page}, - KResult, -}; - -pub trait ClusterReadIterator<'data>: Iterator<Item = KResult<&'data [u8]>> + 'data {} -impl<'a, I> ClusterReadIterator<'a> for I where I: Iterator<Item = KResult<&'a [u8]>> + 'a {} - -pub(super) trait ClusterRead<'data> { - fn read<'vfs>(self, vfs: &'vfs FatFs, offset: usize) -> impl ClusterReadIterator<'data> - where - Self: Sized, - 'vfs: 'data; -} - -impl<'data, 'fat: 'data> ClusterRead<'data> for ClusterIterator<'fat> { - fn read<'vfs: 'data>(self, vfs: &'vfs FatFs, offset: usize) -> impl ClusterReadIterator<'data> { - const SECTOR_SIZE: usize = 512; - - let cluster_size = vfs.sectors_per_cluster as usize * SECTOR_SIZE; - assert!(cluster_size <= 0x1000, "Cluster size is too large"); - - let skip_clusters = offset / cluster_size; - let mut inner_offset = offset % cluster_size; - - // TODO: Use block cache. - let buffer_page = Page::alloc(); - - self.skip(skip_clusters).map(move |cluster| { - vfs.read_cluster(cluster, &buffer_page)?; - let data = unsafe { - // SAFETY: No one could be writing to it. - &buffer_page.as_memblk().as_bytes()[inner_offset..] - }; - inner_offset = 0; - Ok(data) - }) - } -} diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 5d9285ec..c59ee801 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -1,5 +1,4 @@ +// pub mod ext4; pub mod fat32; pub mod procfs; -pub mod shm; pub mod tmpfs; -pub mod ext4; diff --git a/src/fs/procfs.rs b/src/fs/procfs.rs index 2ed24613..5c0f5212 100644 --- a/src/fs/procfs.rs +++ b/src/fs/procfs.rs @@ -1,325 +1,240 @@ -use crate::kernel::constants::{EACCES, ENOTDIR}; -use crate::kernel::task::block_on; -use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::{AtomicMode, Mode}; -use crate::{ - io::Buffer, - kernel::{ - mem::paging::PageBuffer, - vfs::{ - dentry::Dentry, - inode::{define_struct_inode, AtomicIno, Ino, Inode, InodeData}, - mount::{dump_mounts, register_filesystem, Mount, MountCreator}, - vfs::Vfs, - DevId, - }, - }, - prelude::*, -}; -use alloc::sync::{Arc, Weak}; -use core::{ops::ControlFlow, sync::atomic::Ordering}; -use eonix_sync::{AsProof as _, AsProofMut as _, LazyLock, Locked}; -use itertools::Itertools; - -#[allow(dead_code)] -pub trait ProcFsFile: Send + Sync { - fn can_read(&self) -> bool { - false - } +use alloc::sync::Arc; +use core::sync::atomic::{AtomicU64, Ordering}; - fn can_write(&self) -> bool { - false - } +use async_trait::async_trait; +use eonix_sync::{LazyLock, RwLock}; - fn read(&self, _buffer: &mut PageBuffer) -> KResult<usize> { - Err(EACCES) - } +use crate::io::Buffer; +use crate::kernel::constants::{EACCES, EISDIR, ENOTDIR}; +use crate::kernel::mem::paging::PageBuffer; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse}; +use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; +use crate::kernel::vfs::types::{DeviceId, Format, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; +use crate::prelude::*; - fn write(&self, _buffer: &[u8]) -> KResult<usize> { - Err(EACCES) - } +struct Node { + kind: NodeKind, } -pub enum ProcFsNode { - File(Arc<FileInode>), - Dir(Arc<DirInode>), +enum NodeKind { + File(FileInode), + Dir(DirInode), } -impl ProcFsNode { - fn unwrap(&self) -> Arc<dyn Inode> { - match self { - ProcFsNode::File(inode) => inode.clone(), - ProcFsNode::Dir(inode) => inode.clone(), - } - } - - fn ino(&self) -> Ino { - match self { - ProcFsNode::File(inode) => inode.ino, - ProcFsNode::Dir(inode) => inode.ino, - } - } +struct FileInode { + read: Option<Box<dyn Fn(&mut PageBuffer) -> KResult<()> + Send + Sync>>, + // TODO: Implement writes to procfs files + #[allow(unused)] + write: Option<()>, } -define_struct_inode! { - pub struct FileInode { - file: Box<dyn ProcFsFile>, - } +struct DirInode { + entries: RwLock<Vec<(Arc<[u8]>, InodeUse)>>, } -impl FileInode { - pub fn new(ino: Ino, vfs: Weak<ProcFs>, file: Box<dyn ProcFsFile>) -> Arc<Self> { - let mut mode = Mode::REG; - if file.can_read() { - mode.set_perm(0o444); - } - if file.can_write() { - mode.set_perm(0o222); - } +impl InodeOps for Node { + type SuperBlock = ProcFs; - let mut inode = Self { - idata: InodeData::new(ino, vfs), - file, + async fn read( + &self, _: SbUse<Self::SuperBlock>, _: &InodeUse, + buffer: &mut dyn Buffer, offset: usize, + ) -> KResult<usize> { + let NodeKind::File(file_inode) = &self.kind else { + return Err(EISDIR); }; - inode.idata.mode.store(mode); - inode.idata.nlink.store(1, Ordering::Relaxed); - *inode.ctime.get_mut() = Instant::now(); - *inode.mtime.get_mut() = Instant::now(); - *inode.atime.get_mut() = Instant::now(); - - Arc::new(inode) - } -} - -impl Inode for FileInode { - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - if !self.file.can_read() { + let Some(read_fn) = &file_inode.read else { return Err(EACCES); - } + }; let mut page_buffer = PageBuffer::new(); - self.file.read(&mut page_buffer)?; + read_fn(&mut page_buffer)?; - let data = page_buffer - .data() - .split_at_checked(offset) - .map(|(_, data)| data); + let Some((_, data)) = page_buffer.data().split_at_checked(offset) + else { + return Ok(0); + }; - match data { - None => Ok(0), - Some(data) => Ok(buffer.fill(data)?.allow_partial()), - } + Ok(buffer.fill(data)?.allow_partial()) } -} -define_struct_inode! { - pub struct DirInode { - entries: Locked<Vec<(Arc<[u8]>, ProcFsNode)>, ()>, - } -} + async fn lookup( + &self, _: SbUse<Self::SuperBlock>, _: &InodeUse, dentry: &Arc<Dentry>, + ) -> KResult<Option<InodeUse>> { + let NodeKind::Dir(dir) = &self.kind else { + return Err(ENOTDIR); + }; -impl DirInode { - pub fn new(ino: Ino, vfs: Weak<ProcFs>) -> Arc<Self> { - Self::new_locked(ino, vfs, |inode, rwsem| unsafe { - addr_of_mut_field!(inode, entries).write(Locked::new(vec![], rwsem)); - addr_of_mut_field!(&mut *inode, mode).write(AtomicMode::from(Mode::DIR.perm(0o755))); - addr_of_mut_field!(&mut *inode, nlink).write(1.into()); - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } -} + let entries = dir.entries.read().await; -impl Inode for DirInode { - fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<Arc<dyn Inode>>> { - let lock = block_on(self.rwsem.read()); - Ok(self - .entries - .access(lock.prove()) - .iter() - .find_map(|(name, node)| (name == &***dentry.name()).then(|| node.unwrap()))) - } + let dent_name = dentry.name(); + for (name, node) in entries.iter() { + if *name == ***dent_name { + return Ok(Some(node.clone() as _)); + } + } - fn do_readdir( - &self, - offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, - ) -> KResult<usize> { - let lock = block_on(self.rwsem.read()); - self.entries - .access(lock.prove()) - .iter() - .skip(offset) - .map(|(name, node)| callback(name.as_ref(), node.ino())) - .take_while(|result| result.map_or(true, |flow| flow.is_continue())) - .take_while_inclusive(|result| result.is_ok()) - .fold_ok(0, |acc, _| acc + 1) + Ok(None) } -} -impl_any!(ProcFs); -pub struct ProcFs { - root_node: Arc<DirInode>, - next_ino: AtomicIno, -} + async fn readdir( + &self, _: SbUse<Self::SuperBlock>, _: &InodeUse, offset: usize, + callback: &mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> KResult<KResult<usize>> { + let NodeKind::Dir(dir) = &self.kind else { + return Err(ENOTDIR); + }; -impl Vfs for ProcFs { - fn io_blksize(&self) -> usize { - 4096 - } + let entries = dir.entries.read().await; - fn fs_devid(&self) -> DevId { - 10 - } + let mut count = 0; + for (name, node) in entries.iter().skip(offset) { + match callback(name.as_ref(), node.ino) { + Err(err) => return Ok(Err(err)), + Ok(true) => count += 1, + Ok(false) => break, + } + } - fn is_read_only(&self) -> bool { - false + Ok(Ok(count)) + } +} + +impl Node { + pub fn new_file( + ino: Ino, sb: SbRef<ProcFs>, + read: impl Fn(&mut PageBuffer) -> KResult<()> + Send + Sync + 'static, + ) -> InodeUse { + InodeUse::new( + sb, + ino, + Format::REG, + InodeInfo { + size: 0, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o444), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }, + Self { + kind: NodeKind::File(FileInode::new(Box::new(read))), + }, + ) + } + + fn new_dir(ino: Ino, sb: SbRef<ProcFs>) -> InodeUse { + InodeUse::new( + sb, + ino, + Format::DIR, + InodeInfo { + size: 0, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o755), + atime: Instant::UNIX_EPOCH, + ctime: Instant::UNIX_EPOCH, + mtime: Instant::UNIX_EPOCH, + }, + Self { + kind: NodeKind::Dir(DirInode::new()), + }, + ) } } -static GLOBAL_PROCFS: LazyLock<Arc<ProcFs>> = LazyLock::new(|| { - Arc::new_cyclic(|weak: &Weak<ProcFs>| ProcFs { - root_node: DirInode::new(0, weak.clone()), - next_ino: AtomicIno::new(1), - }) -}); - -struct ProcFsMountCreator; - -#[allow(dead_code)] -impl ProcFsMountCreator { - pub fn get() -> Arc<ProcFs> { - GLOBAL_PROCFS.clone() - } - - pub fn get_weak() -> Weak<ProcFs> { - Arc::downgrade(&GLOBAL_PROCFS) +impl FileInode { + fn new( + read: Box<dyn Fn(&mut PageBuffer) -> KResult<()> + Send + Sync>, + ) -> Self { + Self { + read: Some(read), + write: None, + } } } -impl MountCreator for ProcFsMountCreator { - fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { - let vfs = ProcFsMountCreator::get(); - let root_inode = vfs.root_node.clone(); - Mount::new(mp, vfs, root_inode) - } - - fn check_signature(&self, _: &[u8]) -> KResult<bool> { - Ok(true) +impl DirInode { + pub fn new() -> Self { + Self { + entries: RwLock::new(vec![]), + } } } -pub fn root() -> ProcFsNode { - let vfs = ProcFsMountCreator::get(); - let root = vfs.root_node.clone(); - - ProcFsNode::Dir(root) +pub struct ProcFs { + root: InodeUse, + next_ino: AtomicU64, } -pub fn creat( - parent: &ProcFsNode, - name: Arc<[u8]>, - file: Box<dyn ProcFsFile>, -) -> KResult<ProcFsNode> { - let parent = match parent { - ProcFsNode::File(_) => return Err(ENOTDIR), - ProcFsNode::Dir(parent) => parent, - }; - - let fs = ProcFsMountCreator::get(); - let ino = fs.next_ino.fetch_add(1, Ordering::Relaxed); - - let inode = FileInode::new(ino, Arc::downgrade(&fs), file); - - { - let lock = block_on(parent.idata.rwsem.write()); - parent - .entries - .access_mut(lock.prove_mut()) - .push((name, ProcFsNode::File(inode.clone()))); +impl SuperBlock for ProcFs {} +impl ProcFs { + fn assign_ino(&self) -> Ino { + Ino::new(self.next_ino.fetch_add(1, Ordering::Relaxed)) } - - Ok(ProcFsNode::File(inode)) } -#[allow(dead_code)] -pub fn mkdir(parent: &ProcFsNode, name: &[u8]) -> KResult<ProcFsNode> { - let parent = match parent { - ProcFsNode::File(_) => return Err(ENOTDIR), - ProcFsNode::Dir(parent) => parent, - }; - - let fs = ProcFsMountCreator::get(); - let ino = fs.next_ino.fetch_add(1, Ordering::Relaxed); - - let inode = DirInode::new(ino, Arc::downgrade(&fs)); +static GLOBAL_PROCFS: LazyLock<SbUse<ProcFs>> = LazyLock::new(|| { + SbUse::new_cyclic( + SuperBlockInfo { + io_blksize: 4096, + device_id: DeviceId::new(0, 10), + read_only: false, + }, + |sbref| ProcFs { + root: Node::new_dir(Ino::new(0), sbref), + next_ino: AtomicU64::new(1), + }, + ) +}); - parent - .entries - .access_mut(block_on(inode.rwsem.write()).prove_mut()) - .push((Arc::from(name), ProcFsNode::Dir(inode.clone()))); +struct ProcFsMountCreator; - Ok(ProcFsNode::Dir(inode)) -} +#[async_trait] +impl MountCreator for ProcFsMountCreator { + async fn create_mount( + &self, _source: &str, _flags: u64, mp: &Arc<Dentry>, + ) -> KResult<Mount> { + let fs = GLOBAL_PROCFS.clone(); + let root_inode = fs.backend.root.clone(); -struct DumpMountsFile; -impl ProcFsFile for DumpMountsFile { - fn can_read(&self) -> bool { - true + Mount::new(mp, fs, root_inode) } - fn read(&self, buffer: &mut PageBuffer) -> KResult<usize> { - dump_mounts(&mut buffer.get_writer()); - - Ok(buffer.data().len()) + fn check_signature(&self, _: &[u8]) -> KResult<bool> { + Ok(true) } } -pub fn init() { - register_filesystem("procfs", Arc::new(ProcFsMountCreator)).unwrap(); - - creat( - &root(), - Arc::from(b"mounts".as_slice()), - Box::new(DumpMountsFile), - ) - .unwrap(); -} - -pub struct GenericProcFsFile<ReadFn> +pub async fn populate_root<F>(name: Arc<[u8]>, read_fn: F) where - ReadFn: Send + Sync + Fn(&mut PageBuffer) -> KResult<()>, + F: Send + Sync + Fn(&mut PageBuffer) -> KResult<()> + 'static, { - read_fn: Option<ReadFn>, -} + let procfs = &GLOBAL_PROCFS.backend; + let root = &procfs.root.get_priv::<Node>(); -impl<ReadFn> ProcFsFile for GenericProcFsFile<ReadFn> -where - ReadFn: Send + Sync + Fn(&mut PageBuffer) -> KResult<()>, -{ - fn can_read(&self) -> bool { - self.read_fn.is_some() - } + let NodeKind::Dir(root) = &root.kind else { + unreachable!(); + }; - fn read(&self, buffer: &mut PageBuffer) -> KResult<usize> { - self.read_fn.as_ref().ok_or(EACCES)?(buffer).map(|_| buffer.data().len()) - } + let mut entries = root.entries.write().await; + entries.push(( + name, + Node::new_file( + procfs.assign_ino(), + SbRef::from(&GLOBAL_PROCFS), + read_fn, + ), + )); } -pub fn populate_root<F>(name: Arc<[u8]>, read_fn: F) -> KResult<()> -where - F: Send + Sync + Fn(&mut PageBuffer) -> KResult<()> + 'static, -{ - let root = root(); - - creat( - &root, - name, - Box::new(GenericProcFsFile { - read_fn: Some(read_fn), - }), - ) - .map(|_| ()) +pub async fn init() { + register_filesystem("procfs", Arc::new(ProcFsMountCreator)).unwrap(); } diff --git a/src/fs/shm.rs b/src/fs/shm.rs deleted file mode 100644 index 09d36141..00000000 --- a/src/fs/shm.rs +++ /dev/null @@ -1,146 +0,0 @@ -use core::sync::atomic::{AtomicU32, Ordering}; - -use alloc::{collections::btree_map::BTreeMap, sync::Arc}; -use bitflags::bitflags; -use eonix_sync::{LazyLock, Mutex}; - -use crate::{ - fs::tmpfs::{DirectoryInode, FileInode, TmpFs}, - kernel::{constants::ENOSPC, vfs::inode::Mode}, - prelude::KResult, -}; - -bitflags! { - #[derive(Debug, Clone, Copy)] - pub struct ShmFlags: u32 { - /// Create a new segment. If this flag is not used, then shmget() will - /// find the segment associated with key and check to see if the user - /// has permission to access the segment. - const IPC_CREAT = 0o1000; - /// This flag is used with IPC_CREAT to ensure that this call creates - /// the segment. If the segment already exists, the call fails. - const IPC_EXCL = 0o2000; - - /// Attach the segment for read-only access.If this flag is not specified, - /// the segment is attached for read and write access, and the process - /// must have read and write permission for the segment. - const SHM_RDONLY = 0o10000; - /// round attach address to SHMLBA boundary - const SHM_RND = 0o20000; - /// Allow the contents of the segment to be executed. - const SHM_EXEC = 0o100000; - } -} - -pub const IPC_PRIVATE: usize = 0; - -pub struct ShmManager { - tmpfs: Arc<TmpFs>, - root: Arc<DirectoryInode>, - areas: BTreeMap<u32, ShmArea>, -} - -#[repr(C)] -#[derive(Default, Clone, Copy, Debug)] -pub struct IpcPerm { - key: i32, - uid: u32, - gid: u32, - cuid: u32, - cgid: u32, - mode: u16, - seq: u16, -} - -#[repr(C)] -#[derive(Debug, Clone, Copy)] -pub struct ShmIdDs { - // Ownership and permissions - pub shm_perm: IpcPerm, - // Size of segment (bytes). In our system, this must be aligned - pub shm_segsz: usize, - // Last attach time - pub shm_atime: usize, - // Last detach time - pub shm_dtime: usize, - // Creation time/time of last modification via shmctl() - pub shm_ctime: usize, - // PID of creator - pub shm_cpid: usize, - // PID of last shmat(2)/shmdt(2) - pub shm_lpid: usize, - // No. of current attaches - pub shm_nattch: usize, -} - -impl ShmIdDs { - fn new(size: usize, pid: u32) -> Self { - Self { - shm_perm: IpcPerm::default(), - shm_segsz: size, - shm_atime: 0, - shm_dtime: 0, - shm_ctime: 0, // Should set instant now - shm_cpid: pid as usize, - shm_lpid: 0, - shm_nattch: 0, - } - } -} - -#[derive(Debug)] -pub struct ShmArea { - pub area: Arc<FileInode>, - pub shmid_ds: ShmIdDs, -} - -// A big lock here to protect the shared memory area. -// Can be improved with finer-grained locking? -pub static SHM_MANAGER: LazyLock<Mutex<ShmManager>> = - LazyLock::new(|| Mutex::new(ShmManager::new())); - -impl ShmManager { - fn new() -> Self { - let (tmpfs, root) = TmpFs::create(false).expect("should create shm_area successfully"); - Self { - tmpfs, - root, - areas: BTreeMap::new(), - } - } - - pub fn create_shared_area(&self, size: usize, pid: u32, mode: Mode) -> ShmArea { - let ino = self.tmpfs.assign_ino(); - let vfs = Arc::downgrade(&self.tmpfs); - ShmArea { - area: FileInode::new(ino, vfs, size, mode), - shmid_ds: ShmIdDs::new(size, pid), - } - } - - pub fn get(&self, shmid: u32) -> Option<&ShmArea> { - self.areas.get(&shmid) - } - - pub fn insert(&mut self, shmid: u32, area: ShmArea) { - self.areas.insert(shmid, area); - } -} - -pub fn gen_shm_id(key: usize) -> KResult<u32> { - const SHM_MAGIC: u32 = 114514000; - - static NEXT_SHMID: AtomicU32 = AtomicU32::new(0); - - if key == IPC_PRIVATE { - let shmid = NEXT_SHMID.fetch_add(1, Ordering::Relaxed); - - if shmid >= SHM_MAGIC { - return Err(ENOSPC); - } else { - return Ok(shmid); - } - } - - (key as u32).checked_add(SHM_MAGIC).ok_or(ENOSPC) -} diff --git a/src/fs/tmpfs.rs b/src/fs/tmpfs.rs deleted file mode 100644 index 7a5bd52b..00000000 --- a/src/fs/tmpfs.rs +++ /dev/null @@ -1,613 +0,0 @@ -use crate::io::Stream; -use crate::kernel::constants::{EEXIST, EINVAL, EIO, EISDIR, ENOENT, ENOSYS, ENOTDIR}; -use crate::kernel::mem::{CachePage, CachePageStream, PageCache, PageCacheBackend}; -use crate::kernel::task::block_on; -use crate::kernel::timer::Instant; -use crate::kernel::vfs::inode::RenameData; -use crate::kernel::vfs::inode::{AtomicMode, InodeData}; -use crate::{ - io::Buffer, - kernel::vfs::{ - dentry::{dcache, Dentry}, - inode::{define_struct_inode, AtomicIno, Ino, Inode, Mode, WriteOffset}, - mount::{register_filesystem, Mount, MountCreator, MS_RDONLY}, - vfs::Vfs, - DevId, - }, - prelude::*, -}; -use alloc::sync::{Arc, Weak}; -use core::fmt::Debug; -use core::{ops::ControlFlow, sync::atomic::Ordering}; -use eonix_mm::paging::PAGE_SIZE; -use eonix_sync::{AsProof as _, AsProofMut as _, Locked, Mutex, ProofMut}; -use itertools::Itertools; - -fn acquire(vfs: &Weak<dyn Vfs>) -> KResult<Arc<dyn Vfs>> { - vfs.upgrade().ok_or(EIO) -} - -fn astmp(vfs: &Arc<dyn Vfs>) -> &TmpFs { - vfs.as_any() - .downcast_ref::<TmpFs>() - .expect("corrupted tmpfs data structure") -} - -define_struct_inode! { - struct NodeInode { - devid: DevId, - } -} - -impl NodeInode { - fn new(ino: Ino, vfs: Weak<dyn Vfs>, mode: Mode, devid: DevId) -> Arc<Self> { - Self::new_locked(ino, vfs, |inode, _| unsafe { - addr_of_mut_field!(inode, devid).write(devid); - - addr_of_mut_field!(&mut *inode, mode).write(AtomicMode::from(mode)); - addr_of_mut_field!(&mut *inode, nlink).write(1.into()); - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } -} - -impl Inode for NodeInode { - fn devid(&self) -> KResult<DevId> { - Ok(self.devid) - } -} - -define_struct_inode! { - pub(super) struct DirectoryInode { - entries: Locked<Vec<(Arc<[u8]>, Ino)>, ()>, - } -} - -impl DirectoryInode { - fn new(ino: Ino, vfs: Weak<dyn Vfs>, mode: Mode) -> Arc<Self> { - Self::new_locked(ino, vfs, |inode, rwsem| unsafe { - addr_of_mut_field!(inode, entries) - .write(Locked::new(vec![(Arc::from(b".".as_slice()), ino)], rwsem)); - - addr_of_mut_field!(&mut *inode, size).write(1.into()); - addr_of_mut_field!(&mut *inode, mode) - .write(AtomicMode::from(Mode::DIR.perm(mode.non_format_bits()))); - addr_of_mut_field!(&mut *inode, nlink).write(1.into()); // link from `.` to itself - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } - - fn link(&self, name: Arc<[u8]>, file: &dyn Inode, dlock: ProofMut<'_, ()>) { - let now = Instant::now(); - - // SAFETY: Only `unlink` will do something based on `nlink` count - // No need to synchronize here - file.nlink.fetch_add(1, Ordering::Relaxed); - *self.ctime.lock() = now; - - // SAFETY: `rwsem` has done the synchronization - self.size.fetch_add(1, Ordering::Relaxed); - *self.mtime.lock() = now; - - self.entries.access_mut(dlock).push((name, file.ino)); - } - - fn do_unlink( - &self, - file: &Arc<dyn Inode>, - filename: &[u8], - entries: &mut Vec<(Arc<[u8]>, Ino)>, - now: Instant, - decrease_size: bool, - _dir_lock: ProofMut<()>, - _file_lock: ProofMut<()>, - ) -> KResult<()> { - // SAFETY: `file_lock` has done the synchronization - if file.mode.load().is_dir() { - return Err(EISDIR); - } - - entries.retain(|(name, ino)| *ino != file.ino || name.as_ref() != filename); - - if decrease_size { - // SAFETY: `dir_lock` has done the synchronization - self.size.fetch_sub(1, Ordering::Relaxed); - } - - *self.mtime.lock() = now; - - // The last reference to the inode is held by some dentry - // and will be released when the dentry is released - - // SAFETY: `file_lock` has done the synchronization - file.nlink.fetch_sub(1, Ordering::Relaxed); - *file.ctime.lock() = now; - - Ok(()) - } -} - -impl Inode for DirectoryInode { - fn do_readdir( - &self, - offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, - ) -> KResult<usize> { - let lock = block_on(self.rwsem.read()); - self.entries - .access(lock.prove()) - .iter() - .skip(offset) - .map(|(name, ino)| callback(&name, *ino)) - .take_while(|result| result.map_or(true, |flow| flow.is_continue())) - .take_while_inclusive(|result| result.is_ok()) - .fold_ok(0, |acc, _| acc + 1) - } - - fn creat(&self, at: &Arc<Dentry>, mode: Mode) -> KResult<()> { - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let file = FileInode::new(ino, self.vfs.clone(), 0, mode); - - self.link(at.get_name(), file.as_ref(), rwsem.prove_mut()); - at.save_reg(file) - } - - fn mknod(&self, at: &Dentry, mode: Mode, dev: DevId) -> KResult<()> { - if !mode.is_chr() && !mode.is_blk() { - return Err(EINVAL); - } - - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let file = NodeInode::new(ino, self.vfs.clone(), mode, dev); - - self.link(at.get_name(), file.as_ref(), rwsem.prove_mut()); - at.save_reg(file) - } - - fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let file = SymlinkInode::new(ino, self.vfs.clone(), target.into()); - - self.link(at.get_name(), file.as_ref(), rwsem.prove_mut()); - at.save_symlink(file) - } - - fn mkdir(&self, at: &Dentry, mode: Mode) -> KResult<()> { - let vfs = acquire(&self.vfs)?; - let vfs = astmp(&vfs); - - let rwsem = block_on(self.rwsem.write()); - - let ino = vfs.assign_ino(); - let newdir = DirectoryInode::new(ino, self.vfs.clone(), mode); - - self.link(at.get_name(), newdir.as_ref(), rwsem.prove_mut()); - at.save_dir(newdir) - } - - fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { - let _vfs = acquire(&self.vfs)?; - - let dir_lock = block_on(self.rwsem.write()); - - let file = at.get_inode()?; - let filename = at.get_name(); - let file_lock = block_on(file.rwsem.write()); - - let entries = self.entries.access_mut(dir_lock.prove_mut()); - - self.do_unlink( - &file, - &filename, - entries, - Instant::now(), - true, - dir_lock.prove_mut(), - file_lock.prove_mut(), - )?; - - // Remove the dentry from the dentry cache immediately - // so later lookup will fail with ENOENT - dcache::d_remove(at); - - Ok(()) - } - - fn chmod(&self, mode: Mode) -> KResult<()> { - let _vfs = acquire(&self.vfs)?; - let _lock = block_on(self.rwsem.write()); - - // SAFETY: `rwsem` has done the synchronization - let old = self.mode.load(); - self.mode.store(old.perm(mode.non_format_bits())); - *self.ctime.lock() = Instant::now(); - - Ok(()) - } - - fn rename(&self, rename_data: RenameData) -> KResult<()> { - let RenameData { - old_dentry, - new_dentry, - new_parent, - is_exchange, - no_replace, - vfs, - } = rename_data; - - if is_exchange { - println_warn!("TmpFs does not support exchange rename for now"); - return Err(ENOSYS); - } - - let vfs = vfs - .as_any() - .downcast_ref::<TmpFs>() - .expect("vfs must be a TmpFs"); - - let _rename_lock = block_on(vfs.rename_lock.lock()); - - let old_file = old_dentry.get_inode()?; - let new_file = new_dentry.get_inode(); - - if no_replace && new_file.is_ok() { - return Err(EEXIST); - } - - let same_parent = Arc::as_ptr(&new_parent) == &raw const *self; - if same_parent { - // Same directory rename - // Remove from old location and add to new location - let parent_lock = block_on(self.rwsem.write()); - let entries = self.entries.access_mut(parent_lock.prove_mut()); - - fn rename_old( - old_entry: &mut (Arc<[u8]>, Ino), - old_file: &Arc<dyn Inode + 'static>, - new_dentry: &Arc<Dentry>, - now: Instant, - ) { - let (name, _) = old_entry; - *name = new_dentry.get_name(); - *old_file.ctime.lock() = now; - } - - let old_ino = old_file.ino; - let new_ino = new_file.as_ref().ok().map(|f| f.ino); - let old_name = old_dentry.get_name(); - let new_name = new_dentry.get_name(); - - // Find the old and new entries in the directory after we've locked the directory. - let indices = - entries - .iter() - .enumerate() - .fold([None, None], |[old, new], (idx, (name, ino))| { - if Some(*ino) == new_ino && *name == new_name { - [old, Some(idx)] - } else if *ino == old_ino && *name == old_name { - [Some(idx), new] - } else { - [old, new] - } - }); - - let (old_entry_idx, new_entry_idx) = match indices { - [None, ..] => return Err(ENOENT), - [Some(old_idx), new_idx] => (old_idx, new_idx), - }; - - let now = Instant::now(); - - if let Some(new_idx) = new_entry_idx { - // Replace existing file (i.e. rename the old and unlink the new) - let new_file = new_file.unwrap(); - let _new_file_lock = block_on(new_file.rwsem.write()); - - // SAFETY: `new_file_lock` has done the synchronization - match (new_file.mode.load(), old_file.mode.load()) { - (Mode::DIR, _) => return Err(EISDIR), - (_, Mode::DIR) => return Err(ENOTDIR), - _ => {} - } - - entries.remove(new_idx); - - // SAFETY: `parent_lock` has done the synchronization - self.size.fetch_sub(1, Ordering::Relaxed); - - // The last reference to the inode is held by some dentry - // and will be released when the dentry is released - - // SAFETY: `new_file_lock` has done the synchronization - new_file.nlink.fetch_sub(1, Ordering::Relaxed); - *new_file.ctime.lock() = now; - } - - rename_old(&mut entries[old_entry_idx], &old_file, new_dentry, now); - *self.mtime.lock() = now; - } else { - // Cross-directory rename - handle similar to same directory case - - // Get new parent directory - let new_parent_inode = new_dentry.parent().get_inode()?; - assert!(new_parent_inode.is_dir()); - let new_parent = (new_parent_inode.as_ref() as &dyn Any) - .downcast_ref::<DirectoryInode>() - .expect("new parent must be a DirectoryInode"); - - let old_parent_lock = block_on(self.rwsem.write()); - let new_parent_lock = block_on(new_parent_inode.rwsem.write()); - - let old_ino = old_file.ino; - let new_ino = new_file.as_ref().ok().map(|f| f.ino); - let old_name = old_dentry.get_name(); - let new_name = new_dentry.get_name(); - - // Find the old entry in the old directory - let old_entries = self.entries.access_mut(old_parent_lock.prove_mut()); - let old_pos = old_entries - .iter() - .position(|(name, ino)| *ino == old_ino && *name == old_name) - .ok_or(ENOENT)?; - - // Find the new entry in the new directory (if it exists) - let new_entries = new_parent.entries.access_mut(new_parent_lock.prove_mut()); - let has_new = new_entries - .iter() - .position(|(name, ino)| Some(*ino) == new_ino && *name == new_name) - .is_some(); - - let now = Instant::now(); - - if has_new { - // Replace existing file (i.e. move the old and unlink the new) - let new_file = new_file.unwrap(); - let new_file_lock = block_on(new_file.rwsem.write()); - - match (old_file.mode.load(), new_file.mode.load()) { - (Mode::DIR, Mode::DIR) => {} - (Mode::DIR, _) => return Err(ENOTDIR), - (_, _) => {} - } - - // Unlink the old file that was replaced - new_parent.do_unlink( - &new_file, - &new_name, - new_entries, - now, - false, - new_parent_lock.prove_mut(), - new_file_lock.prove_mut(), - )?; - } else { - new_parent.size.fetch_add(1, Ordering::Relaxed); - } - - // Remove from old directory - old_entries.remove(old_pos); - - // Add new entry - new_entries.push((new_name, old_ino)); - - self.size.fetch_sub(1, Ordering::Relaxed); - *self.mtime.lock() = now; - *old_file.ctime.lock() = now; - } - - block_on(dcache::d_exchange(old_dentry, new_dentry)); - - Ok(()) - } -} - -define_struct_inode! { - struct SymlinkInode { - target: Arc<[u8]>, - } -} - -impl SymlinkInode { - fn new(ino: Ino, vfs: Weak<dyn Vfs>, target: Arc<[u8]>) -> Arc<Self> { - Self::new_locked(ino, vfs, |inode, _| unsafe { - let len = target.len(); - addr_of_mut_field!(inode, target).write(target); - - addr_of_mut_field!(&mut *inode, mode).write(AtomicMode::from(Mode::LNK.perm(0o777))); - addr_of_mut_field!(&mut *inode, size).write((len as u64).into()); - addr_of_mut_field!(&mut *inode, ctime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, mtime).write(Spin::new(Instant::now())); - addr_of_mut_field!(&mut *inode, atime).write(Spin::new(Instant::now())); - }) - } -} - -impl Inode for SymlinkInode { - fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { - buffer - .fill(self.target.as_ref()) - .map(|result| result.allow_partial()) - } - - fn chmod(&self, _: Mode) -> KResult<()> { - Ok(()) - } -} - -define_struct_inode! { - pub struct FileInode { - pages: PageCache, - } -} - -impl Debug for FileInode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "FileInode({:?})", self.idata) - } -} - -impl FileInode { - pub fn new(ino: Ino, vfs: Weak<dyn Vfs>, size: usize, mode: Mode) -> Arc<Self> { - let inode = Arc::new_cyclic(|weak_self: &Weak<FileInode>| FileInode { - idata: InodeData::new(ino, vfs), - pages: PageCache::new(weak_self.clone()), - }); - - inode.mode.store(Mode::REG.perm(mode.non_format_bits())); - inode.nlink.store(1, Ordering::Relaxed); - inode.size.store(size as u64, Ordering::Relaxed); - inode - } -} - -impl PageCacheBackend for FileInode { - fn read_page(&self, _cache_page: &mut CachePage, _offset: usize) -> KResult<usize> { - Ok(PAGE_SIZE) - } - - fn write_page(&self, _page: &mut CachePageStream, _offset: usize) -> KResult<usize> { - Ok(PAGE_SIZE) - } - - fn size(&self) -> usize { - self.size.load(Ordering::Relaxed) as usize - } -} - -impl Inode for FileInode { - fn page_cache(&self) -> Option<&PageCache> { - Some(&self.pages) - } - - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - let _lock = block_on(self.rwsem.write()); - block_on(self.pages.read(buffer, offset)) - } - - fn write(&self, stream: &mut dyn Stream, offset: WriteOffset) -> KResult<usize> { - // TODO: We don't need that strong guarantee, find some way to avoid locks - let _lock = block_on(self.rwsem.write()); - - let mut store_new_end = None; - let offset = match offset { - WriteOffset::Position(offset) => offset, - WriteOffset::End(end) => { - store_new_end = Some(end); - - // SAFETY: `lock` has done the synchronization - self.size.load(Ordering::Relaxed) as usize - } - }; - - let wrote = block_on(self.pages.write(stream, offset))?; - let cursor_end = offset + wrote; - - if let Some(store_end) = store_new_end { - *store_end = cursor_end; - } - - // SAFETY: `lock` has done the synchronization - *self.mtime.lock() = Instant::now(); - self.size.store(cursor_end as u64, Ordering::Relaxed); - - Ok(wrote) - } - - fn truncate(&self, length: usize) -> KResult<()> { - let _lock = block_on(self.rwsem.write()); - block_on(self.pages.resize(length))?; - self.size.store(length as u64, Ordering::Relaxed); - *self.mtime.lock() = Instant::now(); - Ok(()) - } - - fn chmod(&self, mode: Mode) -> KResult<()> { - let _vfs = acquire(&self.vfs)?; - let _lock = block_on(self.rwsem.write()); - - // SAFETY: `rwsem` has done the synchronization - let old = self.mode.load(); - self.mode.store(old.perm(mode.non_format_bits())); - *self.ctime.lock() = Instant::now(); - - Ok(()) - } -} - -impl_any!(TmpFs); -pub(super) struct TmpFs { - next_ino: AtomicIno, - readonly: bool, - rename_lock: Mutex<()>, -} - -impl Vfs for TmpFs { - fn io_blksize(&self) -> usize { - 4096 - } - - fn fs_devid(&self) -> DevId { - 2 - } - - fn is_read_only(&self) -> bool { - self.readonly - } -} - -impl TmpFs { - pub(super) fn assign_ino(&self) -> Ino { - self.next_ino.fetch_add(1, Ordering::AcqRel) - } - - pub fn create(readonly: bool) -> KResult<(Arc<TmpFs>, Arc<DirectoryInode>)> { - let tmpfs = Arc::new(Self { - next_ino: AtomicIno::new(1), - readonly, - rename_lock: Mutex::new(()), - }); - - let weak = Arc::downgrade(&tmpfs); - let root_dir = DirectoryInode::new(0, weak, Mode::new(0o755)); - - Ok((tmpfs, root_dir)) - } -} - -struct TmpFsMountCreator; - -impl MountCreator for TmpFsMountCreator { - fn create_mount(&self, _source: &str, flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { - let (fs, root_inode) = TmpFs::create(flags & MS_RDONLY != 0)?; - - Mount::new(mp, fs, root_inode) - } - - fn check_signature(&self, _: &[u8]) -> KResult<bool> { - Ok(true) - } -} - -pub fn init() { - register_filesystem("tmpfs", Arc::new(TmpFsMountCreator)).unwrap(); -} diff --git a/src/fs/tmpfs/dir.rs b/src/fs/tmpfs/dir.rs new file mode 100644 index 00000000..4dd64d52 --- /dev/null +++ b/src/fs/tmpfs/dir.rs @@ -0,0 +1,404 @@ +use alloc::sync::Arc; +use alloc::vec; +use alloc::vec::Vec; + +use eonix_log::println_warn; +use eonix_sync::{LazyLock, RwLock}; + +use super::file::{DeviceInode, FileInode, SymlinkInode}; +use super::TmpFs; +use crate::kernel::constants::{EEXIST, EINVAL, EISDIR, ENOENT, ENOSYS, ENOTDIR}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::dentry::{dcache, Dentry}; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse, RenameData}; +use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; +use crate::kernel::vfs::{SbRef, SbUse}; +use crate::prelude::KResult; + +pub struct DirectoryInode { + entries: RwLock<Vec<(Arc<[u8]>, Ino)>>, +} + +fn link(dir: &InodeUse, entries: &mut Vec<(Arc<[u8]>, Ino)>, name: Arc<[u8]>, file: &InodeUse) { + let mut dir_info = dir.info.lock(); + let mut file_info = file.info.lock(); + + let now = Instant::now(); + + file_info.nlink += 1; + file_info.ctime = now; + + dir_info.size += 1; + dir_info.mtime = now; + dir_info.ctime = now; + + entries.push((name, file.ino)); +} + +impl DirectoryInode { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, perm: Permission) -> InodeUse { + static DOT: LazyLock<Arc<[u8]>> = LazyLock::new(|| Arc::from(b".".as_slice())); + + let now = Instant::now(); + + InodeUse::new( + sb, + ino, + Format::DIR, + InodeInfo { + size: 1, + nlink: 1, // link from `.` to itself + perm, + ctime: now, + mtime: now, + atime: now, + uid: 0, + gid: 0, + }, + Self { + entries: RwLock::new(vec![(DOT.clone(), ino)]), + }, + ) + } + + fn do_unlink( + &self, + file: &InodeUse, + filename: &[u8], + entries: &mut Vec<(Arc<[u8]>, Ino)>, + now: Instant, + decrease_size: bool, + self_info: &mut InodeInfo, + file_info: &mut InodeInfo, + ) -> KResult<()> { + // SAFETY: `file_lock` has done the synchronization + if file.format == Format::DIR { + return Err(EISDIR); + } + + let file_ino = file.ino; + entries.retain(|(name, ino)| *ino != file_ino || name.as_ref() != filename); + + if decrease_size { + self_info.size -= 1; + } + + self_info.mtime = now; + self_info.ctime = now; + + // The last reference to the inode is held by some dentry + // and will be released when the dentry is released + + file_info.nlink -= 1; + file_info.ctime = now; + + // TODO!!!: Remove the file if nlink == 1 + + Ok(()) + } +} + +impl InodeOps for DirectoryInode { + type SuperBlock = TmpFs; + + async fn readdir( + &self, + sb: SbUse<Self::SuperBlock>, + _: &InodeUse, + offset: usize, + for_each_entry: &mut (dyn FnMut(&[u8], Ino) -> KResult<bool> + Send), + ) -> KResult<KResult<usize>> { + let _sb = sb; + let entries = self.entries.read().await; + + let mut count = 0; + for entry in entries.iter().skip(offset) { + match for_each_entry(&entry.0, entry.1) { + Err(err) => return Ok(Err(err)), + Ok(false) => break, + Ok(true) => count += 1, + } + } + + Ok(Ok(count)) + } + + async fn create( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Arc<Dentry>, + perm: Permission, + ) -> KResult<()> { + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let file = FileInode::new(ino, sb.get_ref(), 0, perm); + + link(inode, &mut entries, at.get_name(), &file); + at.fill(file); + + Ok(()) + } + + async fn mknod( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Dentry, + mode: Mode, + dev: DeviceId, + ) -> KResult<()> { + if !mode.is_chr() && !mode.is_blk() { + return Err(EINVAL); + } + + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let file = DeviceInode::new(ino, sb.get_ref(), mode, dev); + + link(inode, &mut entries, at.get_name(), &file); + at.fill(file); + + Ok(()) + } + + async fn symlink( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Arc<Dentry>, + target: &[u8], + ) -> KResult<()> { + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let file = SymlinkInode::new(ino, sb.get_ref(), target.into()); + + link(inode, &mut entries, at.get_name(), &file); + at.fill(file); + + Ok(()) + } + + async fn mkdir( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Dentry, + perm: Permission, + ) -> KResult<()> { + let mut entries = self.entries.write().await; + + let ino = sb.backend.assign_ino(); + let new_dir = DirectoryInode::new(ino, sb.get_ref(), perm); + + link(inode, &mut entries, at.get_name(), &new_dir); + at.fill(new_dir); + + Ok(()) + } + + async fn unlink( + &self, + _sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + at: &Arc<Dentry>, + ) -> KResult<()> { + let mut entries = self.entries.write().await; + + let file = at.get_inode()?; + let filename = at.get_name(); + + self.do_unlink( + &file, + &filename, + &mut entries, + Instant::now(), + true, + &mut inode.info.lock(), + &mut file.info.lock(), + )?; + + // Remove the dentry from the dentry cache immediately + // so later lookup will fail with ENOENT + dcache::d_remove(at); + + Ok(()) + } + + async fn rename( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + rename_data: RenameData<'_, '_>, + ) -> KResult<()> { + let _rename_lock = sb.backend.rename_lock.lock().await; + let mut self_entries = self.entries.write().await; + + let RenameData { + old_dentry, + new_dentry, + new_parent, + is_exchange, + no_replace, + } = rename_data; + + if is_exchange { + println_warn!("TmpFs does not support exchange rename for now"); + return Err(ENOSYS); + } + + let old_file = old_dentry.get_inode()?; + let new_file = new_dentry.inode(); + + if no_replace && new_file.is_some() { + return Err(EEXIST); + } + + if inode == &new_parent { + // Same directory rename + // Remove from old location and add to new location + let old_ino = old_file.ino; + let new_ino = new_file.as_ref().map(|f| f.ino); + let old_name = old_dentry.get_name(); + let new_name = new_dentry.get_name(); + + // Find the old and new entries in the directory after we've locked the directory. + let (mut old_ent_idx, mut new_ent_idx) = (None, None); + for (idx, (name, ino)) in self_entries.iter().enumerate() { + if *ino == old_ino && *name == old_name { + old_ent_idx = Some(idx); + } + + if Some(*ino) == new_ino && *name == new_name { + new_ent_idx = Some(idx); + } + } + + let Some(old_ent_idx) = old_ent_idx else { + return Err(ENOENT); + }; + + if Some(old_ent_idx) == new_ent_idx { + return Ok(()); + } + + let now = Instant::now(); + if let Some(new_idx) = new_ent_idx { + // Replace existing file (i.e. rename the old and unlink the new) + let new_file = new_file.unwrap(); + + match (new_file.format, old_file.format) { + (Format::DIR, _) => return Err(EISDIR), + (_, Format::DIR) => return Err(ENOTDIR), + _ => {} + } + + self_entries.remove(new_idx); + + inode.info.lock().size -= 1; + + // The last reference to the inode is held by some dentry + // and will be released when the dentry is released + + let mut new_info = new_file.info.lock(); + + new_info.nlink -= 1; + new_info.mtime = now; + new_info.ctime = now; + } + + let (name, _) = &mut self_entries[old_ent_idx]; + *name = new_dentry.get_name(); + + let mut self_info = inode.info.lock(); + self_info.mtime = now; + self_info.ctime = now; + } else { + // Cross-directory rename - handle similar to same directory case + + // Get new parent directory + let new_parent = new_dentry.parent().get_inode()?; + assert_eq!(new_parent.format, Format::DIR); + + let new_parent_priv = new_parent.get_priv::<DirectoryInode>(); + let mut new_entries = new_parent_priv.entries.write().await; + + let old_ino = old_file.ino; + let new_ino = new_file.as_ref().map(|f| f.ino); + let old_name = old_dentry.get_name(); + let new_name = new_dentry.get_name(); + + // Find the old entry in the old directory + let old_pos = self_entries + .iter() + .position(|(name, ino)| *ino == old_ino && *name == old_name) + .ok_or(ENOENT)?; + + // Find the new entry in the new directory (if it exists) + let has_new = new_entries + .iter() + .position(|(name, ino)| Some(*ino) == new_ino && *name == new_name) + .is_some(); + + let now = Instant::now(); + + if has_new { + // Replace existing file (i.e. move the old and unlink the new) + let new_file = new_file.unwrap(); + + match (old_file.format, new_file.format) { + (Format::DIR, Format::DIR) => {} + (Format::DIR, _) => return Err(ENOTDIR), + (_, _) => {} + } + + // Unlink the old file that was replaced + new_parent_priv.do_unlink( + &new_file, + &new_name, + &mut new_entries, + now, + false, + &mut new_parent.info.lock(), + &mut new_file.info.lock(), + )?; + } else { + let mut info = new_parent.info.lock(); + + info.size += 1; + info.mtime = now; + info.ctime = now; + } + + // Remove from old directory + self_entries.remove(old_pos); + + // Add new entry + new_entries.push((new_name, old_ino)); + + let mut self_info = inode.info.lock(); + self_info.size -= 1; + self_info.mtime = now; + self_info.ctime = now; + } + + dcache::d_exchange(old_dentry, new_dentry).await; + Ok(()) + } + + async fn chmod( + &self, + _sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + perm: Permission, + ) -> KResult<()> { + let mut info = inode.info.lock(); + info.perm = perm; + info.ctime = Instant::now(); + + Ok(()) + } +} diff --git a/src/fs/tmpfs/file.rs b/src/fs/tmpfs/file.rs new file mode 100644 index 00000000..aafae539 --- /dev/null +++ b/src/fs/tmpfs/file.rs @@ -0,0 +1,267 @@ +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::Arc; + +use super::TmpFs; +use crate::io::{Buffer, Stream}; +use crate::kernel::mem::{CachePage, PageCache, PageOffset}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::inode::{Ino, InodeInfo, InodeOps, InodeUse, WriteOffset}; +use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; +use crate::kernel::vfs::{SbRef, SbUse}; +use crate::prelude::KResult; + +pub struct FileInode; + +impl FileInode { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, size: usize, perm: Permission) -> InodeUse { + let now = Instant::now(); + + InodeUse::new( + sb, + ino, + Format::REG, + InodeInfo { + size: size as _, + nlink: 1, + uid: 0, + gid: 0, + perm, + atime: now, + ctime: now, + mtime: now, + }, + Self, + ) + } +} + +impl InodeOps for FileInode { + type SuperBlock = TmpFs; + + async fn read( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + buffer: &mut dyn Buffer, + offset: usize, + ) -> KResult<usize> { + let _lock = inode.rwsem.read().await; + inode.get_page_cache().read(buffer, offset).await + } + + async fn write( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + stream: &mut dyn Stream, + offset: WriteOffset<'_>, + ) -> KResult<usize> { + let _lock = inode.rwsem.write().await; + + let mut store_new_end = None; + let offset = match offset { + WriteOffset::Position(offset) => offset, + WriteOffset::End(end) => { + store_new_end = Some(end); + + // `info.size` won't change since we are holding the write lock. + inode.info.lock().size as usize + } + }; + + let page_cache = inode.get_page_cache(); + + if Arc::strong_count(&page_cache) == 1 { + // XXX: A temporary workaround here. Change this ASAP... + // Prevent the page cache from being dropped during the write. + let _ = Arc::into_raw(page_cache.clone()); + } + + let wrote = page_cache.write(stream, offset).await?; + let cursor_end = offset + wrote; + + if let Some(store_end) = store_new_end { + *store_end = cursor_end; + } + + Ok(wrote) + } + + async fn truncate( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + length: usize, + ) -> KResult<()> { + let _lock = inode.rwsem.write().await; + + let now = Instant::now(); + let mut info = inode.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = length as u64; + + Ok(()) + } + + async fn chmod( + &self, + _sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + perm: Permission, + ) -> KResult<()> { + let mut info = inode.info.lock(); + + info.perm = perm; + info.ctime = Instant::now(); + + Ok(()) + } + + async fn read_page( + &self, + _: SbUse<Self::SuperBlock>, + _: &InodeUse, + page: &mut CachePage, + _: PageOffset, + ) -> KResult<()> { + page.lock().as_bytes_mut().fill(0); + Ok(()) + } + + async fn write_page( + &self, + _: SbUse<Self::SuperBlock>, + _: &InodeUse, + _: &mut CachePage, + _: PageOffset, + ) -> KResult<()> { + // XXX: actually we should refuse to do the writeback. + // think of a way to inform that of the page cache. + Ok(()) + } + + async fn write_begin<'a>( + &self, + _: SbUse<Self::SuperBlock>, + _: &InodeUse, + page_cache: &PageCache, + pages: &'a mut BTreeMap<PageOffset, CachePage>, + offset: usize, + _: usize, + ) -> KResult<&'a mut CachePage> { + // TODO: Remove dependency on `page_cache`. + page_cache + .get_page_locked(pages, PageOffset::from_byte_floor(offset)) + .await + } + + async fn write_end( + &self, + _: SbUse<Self::SuperBlock>, + inode: &InodeUse, + _: &PageCache, + _: &mut BTreeMap<PageOffset, CachePage>, + offset: usize, + _: usize, + copied: usize, + ) -> KResult<()> { + let now = Instant::now(); + let mut info = inode.info.lock(); + info.mtime = now; + info.ctime = now; + info.size = info.size.max((offset + copied) as u64); + + Ok(()) + } +} + +pub struct DeviceInode { + devid: DeviceId, +} + +impl DeviceInode { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, mode: Mode, devid: DeviceId) -> InodeUse { + let now = Instant::now(); + + InodeUse::new( + sb, + ino, + mode.format(), + InodeInfo { + size: 0, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(mode.non_format_bits()), + atime: now, + ctime: now, + mtime: now, + }, + Self { devid }, + ) + } +} + +impl InodeOps for DeviceInode { + type SuperBlock = TmpFs; + + async fn chmod( + &self, + _sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + perm: Permission, + ) -> KResult<()> { + let mut info = inode.info.lock(); + info.perm = perm; + info.ctime = Instant::now(); + + Ok(()) + } + + fn devid(&self, _: SbUse<Self::SuperBlock>, _: &InodeUse) -> KResult<DeviceId> { + Ok(self.devid) + } +} + +pub struct SymlinkInode { + target: Arc<[u8]>, +} + +impl SymlinkInode { + pub fn new(ino: Ino, sb: SbRef<TmpFs>, target: Arc<[u8]>) -> InodeUse { + let now = Instant::now(); + + InodeUse::new( + sb, + ino, + Format::LNK, + InodeInfo { + size: target.len() as _, + nlink: 1, + uid: 0, + gid: 0, + perm: Permission::new(0o777), + atime: now, + ctime: now, + mtime: now, + }, + Self { target }, + ) + } +} + +impl InodeOps for SymlinkInode { + type SuperBlock = TmpFs; + + async fn readlink( + &self, + _sb: SbUse<Self::SuperBlock>, + _inode: &InodeUse, + buffer: &mut dyn Buffer, + ) -> KResult<usize> { + buffer + .fill(self.target.as_ref()) + .map(|result| result.allow_partial()) + } +} diff --git a/src/fs/tmpfs/mod.rs b/src/fs/tmpfs/mod.rs new file mode 100644 index 00000000..62a0dfc2 --- /dev/null +++ b/src/fs/tmpfs/mod.rs @@ -0,0 +1,70 @@ +mod dir; +mod file; + +use alloc::sync::Arc; +use core::sync::atomic::{AtomicU64, Ordering}; + +use async_trait::async_trait; +use dir::DirectoryInode; +use eonix_sync::Mutex; + +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{Ino, InodeUse}; +use crate::kernel::vfs::mount::{register_filesystem, Mount, MountCreator}; +use crate::kernel::vfs::types::{DeviceId, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock, SuperBlockInfo}; +use crate::prelude::*; + +pub struct TmpFs { + next_ino: AtomicU64, + rename_lock: Mutex<()>, +} + +impl SuperBlock for TmpFs {} + +impl TmpFs { + fn assign_ino(&self) -> Ino { + Ino::new(self.next_ino.fetch_add(1, Ordering::Relaxed)) + } + + fn create() -> KResult<(SbUse<TmpFs>, InodeUse)> { + let tmpfs = SbUse::new( + SuperBlockInfo { + io_blksize: 4096, + device_id: DeviceId::new(0, 2), + read_only: false, + }, + Self { + next_ino: AtomicU64::new(1), + rename_lock: Mutex::new(()), + }, + ); + + let root_dir = DirectoryInode::new( + tmpfs.backend.assign_ino(), + SbRef::from(&tmpfs), + Permission::new(0o755), + ); + + Ok((tmpfs, root_dir)) + } +} + +struct TmpFsMountCreator; + +#[async_trait] +impl MountCreator for TmpFsMountCreator { + async fn create_mount(&self, _source: &str, _flags: u64, mp: &Arc<Dentry>) -> KResult<Mount> { + let (fs, root_inode) = TmpFs::create()?; + + Mount::new(mp, fs, root_inode) + } + + fn check_signature(&self, _: &[u8]) -> KResult<bool> { + Ok(true) + } +} + +pub fn init() { + register_filesystem("tmpfs", Arc::new(TmpFsMountCreator)).unwrap(); +} diff --git a/src/io.rs b/src/io.rs index 85675dea..1fa79817 100644 --- a/src/io.rs +++ b/src/io.rs @@ -1,6 +1,8 @@ +use core::mem::MaybeUninit; +use core::ops::{Add, AddAssign, Sub}; + use crate::kernel::constants::EFAULT; use crate::prelude::*; -use core::{cmp, mem::MaybeUninit}; #[must_use] #[derive(Debug)] @@ -50,7 +52,9 @@ pub trait Buffer: Send { } pub trait Stream: Send { - fn poll_data<'a>(&mut self, buf: &'a mut [u8]) -> KResult<Option<&'a mut [u8]>>; + fn poll_data<'a>( + &mut self, buf: &'a mut [u8], + ) -> KResult<Option<&'a mut [u8]>>; fn ignore(&mut self, len: usize) -> KResult<Option<usize>>; } @@ -62,9 +66,7 @@ pub trait IntoStream { pub trait StreamRead { fn read_till_end( - &mut self, - buffer: &mut [u8], - func: impl Fn(&mut [u8]) -> KResult<()>, + &mut self, buffer: &mut [u8], func: impl Fn(&mut [u8]) -> KResult<()>, ) -> KResult<usize>; fn ignore_all(&mut self) -> KResult<usize>; @@ -75,9 +77,7 @@ where T: Stream + ?Sized, { fn read_till_end( - &mut self, - buffer: &mut [u8], - func: impl Fn(&mut [u8]) -> KResult<()>, + &mut self, buffer: &mut [u8], func: impl Fn(&mut [u8]) -> KResult<()>, ) -> KResult<usize> { let mut total = 0; while let Some(data) = self.poll_data(buffer)? { @@ -141,7 +141,10 @@ impl<'lt, T: Copy + Sized> UninitBuffer<'lt, T> { Self { data, buffer: ByteBuffer::from(unsafe { - core::slice::from_raw_parts_mut(ptr as *mut u8, core::mem::size_of::<T>()) + core::slice::from_raw_parts_mut( + ptr as *mut u8, + core::mem::size_of::<T>(), + ) }), } } @@ -236,18 +239,26 @@ impl Buffer for ByteBuffer<'_> { } } +pub trait Integer: + Add<Output = Self> + Sub<Output = Self> + AddAssign + Copy + PartialOrd + Ord +{ +} + +impl Integer for u64 {} +impl Integer for usize {} + /// Iterator that generates chunks of a given length from a start index /// until the end of the total length. /// /// The iterator returns a tuple of (start, len) for each chunk. -pub struct Chunks { - end: usize, - cur: usize, - chunk_len: usize, +pub struct Chunks<T: Integer> { + end: T, + cur: T, + chunk_len: T, } -impl Chunks { - pub const fn new(start: usize, total_len: usize, chunk_len: usize) -> Self { +impl<T: Integer> Chunks<T> { + pub fn new(start: T, total_len: T, chunk_len: T) -> Self { Self { end: start + total_len, cur: start, @@ -256,8 +267,8 @@ impl Chunks { } } -impl Iterator for Chunks { - type Item = (usize, usize); +impl<T: Integer> Iterator for Chunks<T> { + type Item = (T, T); fn next(&mut self) -> Option<Self::Item> { if self.cur >= self.end { @@ -265,7 +276,7 @@ impl Iterator for Chunks { } let start = self.cur; - let len = cmp::min(self.chunk_len, self.end - start); + let len = self.chunk_len.min(self.end - start); self.cur += self.chunk_len; Some((start, len)) @@ -284,7 +295,9 @@ impl<'a> ByteStream<'a> { } impl<'a> Stream for ByteStream<'a> { - fn poll_data<'b>(&mut self, buf: &'b mut [u8]) -> KResult<Option<&'b mut [u8]>> { + fn poll_data<'b>( + &mut self, buf: &'b mut [u8], + ) -> KResult<Option<&'b mut [u8]>> { if self.cur >= self.data.len() { return Ok(None); } @@ -320,3 +333,23 @@ impl<'a> IntoStream for &'a [u8] { ByteStream::new(self) } } + +macro_rules! buf_write { + ($buf:ident, $($args:expr),*) => {{ + use core::fmt::Write; + use $crate::io::Buffer; + write!($buf.get_writer(), $($args),*) + .map_err(|_| $crate::kernel::constants::ERANGE) + }}; +} + +macro_rules! buf_writeln { + ($buf:ident, $($args:expr),*) => {{ + use core::fmt::Write; + use $crate::io::Buffer; + writeln!($buf.get_writer(), $($args),*) + .map_err(|_| $crate::kernel::constants::ERANGE) + }}; +} + +pub(crate) use {buf_write, buf_writeln}; diff --git a/src/kernel.rs b/src/kernel.rs index 28d69626..e5712856 100644 --- a/src/kernel.rs +++ b/src/kernel.rs @@ -12,8 +12,12 @@ pub mod user; pub mod vfs; mod chardev; +mod late_init; +mod power; mod terminal; #[allow(unused_imports)] pub use chardev::{CharDevice, CharDeviceType, VirtualCharDevice}; +pub use late_init::{run_late_init, run_late_init_async}; +pub use power::shutdown_system; pub use terminal::{Terminal, TerminalDevice}; diff --git a/src/kernel/block.rs b/src/kernel/block.rs index 349e3656..be2146f8 100644 --- a/src/kernel/block.rs +++ b/src/kernel/block.rs @@ -1,25 +1,18 @@ mod mbr; -use super::{ - constants::ENOENT, - mem::{paging::Page, AsMemoryBlock as _}, - vfs::DevId, -}; -use crate::kernel::constants::{EEXIST, EINVAL}; -use crate::{ - io::{Buffer, FillResult}, - prelude::*, -}; -use alloc::{ - collections::btree_map::{BTreeMap, Entry}, - sync::Arc, -}; +use alloc::collections::btree_map::{BTreeMap, Entry}; +use alloc::sync::Arc; use core::cmp::Ordering; + +use async_trait::async_trait; use mbr::MBRPartTable; -pub fn make_device(major: u32, minor: u32) -> DevId { - (major << 8) & 0xff00u32 | minor & 0xffu32 -} +use super::constants::ENOENT; +use super::mem::Folio; +use super::vfs::types::DeviceId; +use crate::io::{Buffer, Chunks, FillResult}; +use crate::kernel::constants::{EEXIST, EINVAL}; +use crate::prelude::*; pub struct Partition { pub lba_offset: u64, @@ -30,11 +23,12 @@ pub trait PartTable { fn partitions(&self) -> impl Iterator<Item = Partition> + use<'_, Self>; } +#[async_trait] pub trait BlockRequestQueue: Send + Sync { /// Maximum number of sectors that can be read in one request fn max_request_pages(&self) -> u64; - fn submit(&self, req: BlockDeviceRequest) -> KResult<()>; + async fn submit<'a>(&'a self, req: BlockDeviceRequest<'a>) -> KResult<()>; } enum BlockDeviceType { @@ -42,7 +36,7 @@ enum BlockDeviceType { queue: Arc<dyn BlockRequestQueue>, }, Partition { - disk_dev: DevId, + disk_dev: DeviceId, lba_offset: u64, queue: Arc<dyn BlockRequestQueue>, }, @@ -50,7 +44,7 @@ enum BlockDeviceType { pub struct BlockDevice { /// Unique device identifier, major and minor numbers - devid: DevId, + devid: DeviceId, /// Total size of the device in sectors (512 bytes each) sector_count: u64, @@ -77,11 +71,11 @@ impl Ord for BlockDevice { } } -static BLOCK_DEVICE_LIST: Spin<BTreeMap<DevId, Arc<BlockDevice>>> = Spin::new(BTreeMap::new()); +static BLOCK_DEVICE_LIST: Spin<BTreeMap<DeviceId, Arc<BlockDevice>>> = Spin::new(BTreeMap::new()); impl BlockDevice { pub fn register_disk( - devid: DevId, + devid: DeviceId, size: u64, queue: Arc<dyn BlockRequestQueue>, ) -> KResult<Arc<Self>> { @@ -97,13 +91,13 @@ impl BlockDevice { } } - pub fn get(devid: DevId) -> KResult<Arc<Self>> { + pub fn get(devid: DeviceId) -> KResult<Arc<Self>> { BLOCK_DEVICE_LIST.lock().get(&devid).cloned().ok_or(ENOENT) } } impl BlockDevice { - pub fn devid(&self) -> DevId { + pub fn devid(&self) -> DeviceId { self.devid } @@ -121,7 +115,7 @@ impl BlockDevice { }; let device = Arc::new(BlockDevice { - devid: make_device(self.devid >> 8, (self.devid & 0xff) + idx as u32 + 1), + devid: DeviceId::new(self.devid.major, self.devid.minor + idx as u16 + 1), sector_count: size, dev_type: BlockDeviceType::Partition { disk_dev: self.devid, @@ -159,7 +153,7 @@ impl BlockDevice { /// - `req.sector` must be within the disk size /// - `req.buffer` must be enough to hold the data /// - pub fn commit_request(&self, mut req: BlockDeviceRequest) -> KResult<()> { + pub async fn commit_request(&self, mut req: BlockDeviceRequest<'_>) -> KResult<()> { // Verify the request parameters. match &mut req { BlockDeviceRequest::Read { sector, count, .. } => { @@ -184,7 +178,7 @@ impl BlockDevice { } } - self.queue().submit(req) + self.queue().submit(req).await } /// Read some from the block device, may involve some copy and fragmentation @@ -194,178 +188,73 @@ impl BlockDevice { /// # Arguments /// `offset` - offset in bytes /// - pub fn read_some(&self, offset: usize, buffer: &mut dyn Buffer) -> KResult<FillResult> { - let mut sector_start = offset as u64 / 512; - let mut first_sector_offset = offset as u64 % 512; - let mut sector_count = (first_sector_offset + buffer.total() as u64 + 511) / 512; - - let mut nfilled = 0; - 'outer: while sector_count != 0 { - let pages: &[Page]; - let page: Option<Page>; - let page_vec: Option<Vec<Page>>; - - let nread; - - match sector_count { - count if count <= 8 => { - nread = count; - - let _page = Page::alloc(); - page = Some(_page); - pages = core::slice::from_ref(page.as_ref().unwrap()); + pub async fn read_some(&self, offset: usize, buffer: &mut dyn Buffer) -> KResult<FillResult> { + let sector_start = offset as u64 / 512; + let mut first_sector_offset = offset % 512; + let nr_sectors = (first_sector_offset + buffer.total() + 511) / 512; + + let nr_sectors_per_batch = self.queue().max_request_pages() / 2 * 2 * 8; + + let mut nr_filled = 0; + for (start, nr_batch) in Chunks::new(sector_start, nr_sectors as u64, nr_sectors_per_batch) + { + let (page_slice, page, mut page_vec); + match nr_batch { + ..=8 => { + page = Folio::alloc(); + page_slice = core::slice::from_ref(&page); } - count if count <= 16 => { - nread = count; - - let _pages = Page::alloc_order(1); - page = Some(_pages); - pages = core::slice::from_ref(page.as_ref().unwrap()); + ..=16 => { + page = Folio::alloc_order(1); + page_slice = core::slice::from_ref(&page); + } + ..=32 => { + page = Folio::alloc_order(2); + page_slice = core::slice::from_ref(&page); } count => { - nread = count.min(self.queue().max_request_pages()); + let nr_huge_pages = count as usize / 32; + let nr_small_pages = ((count as usize % 32) + 7) / 8; - let npages = (nread + 15) / 16; - let mut _page_vec = Vec::with_capacity(npages as usize); - for _ in 0..npages { - _page_vec.push(Page::alloc_order(1)); - } - page_vec = Some(_page_vec); - pages = page_vec.as_ref().unwrap().as_slice(); + let nr_pages = nr_huge_pages + nr_small_pages; + page_vec = Vec::with_capacity(nr_pages); + + page_vec.resize_with(nr_huge_pages, || Folio::alloc_order(2)); + page_vec.resize_with(nr_pages, || Folio::alloc()); + page_slice = &page_vec; } } let req = BlockDeviceRequest::Read { - sector: sector_start, - count: nread, - buffer: &pages, + sector: start, + count: nr_batch, + buffer: page_slice, }; - self.commit_request(req)?; + self.commit_request(req).await?; - for page in pages.iter() { - // SAFETY: We are the only owner of the page so no one could be mutating it. - let data = unsafe { &page.as_memblk().as_bytes()[first_sector_offset as usize..] }; + for page in page_slice { + let pg = page.lock(); + let data = &pg.as_bytes()[first_sector_offset..]; first_sector_offset = 0; - match buffer.fill(data)? { - FillResult::Done(n) => nfilled += n, - FillResult::Partial(n) => { - nfilled += n; - break 'outer; - } - FillResult::Full => { - break 'outer; - } - } - } - - sector_start += nread; - sector_count -= nread; - } - - if nfilled == buffer.total() { - Ok(FillResult::Done(nfilled)) - } else { - Ok(FillResult::Partial(nfilled)) - } - } - - /// Write some data to the block device, may involve some copy and fragmentation - /// - /// # Arguments - /// `offset` - offset in bytes - /// `data` - data to write - /// - pub fn write_some(&self, offset: usize, data: &[u8]) -> KResult<usize> { - let mut sector_start = offset as u64 / 512; - let mut first_sector_offset = offset as u64 % 512; - let mut remaining_data = data; - let mut nwritten = 0; - - while !remaining_data.is_empty() { - let pages: &[Page]; - let page: Option<Page>; - let page_vec: Option<Vec<Page>>; - - // Calculate sectors needed for this write - let write_end = first_sector_offset + remaining_data.len() as u64; - let sector_count = ((write_end + 511) / 512).min(self.queue().max_request_pages()); - - match sector_count { - count if count <= 8 => { - let _page = Page::alloc(); - page = Some(_page); - pages = core::slice::from_ref(page.as_ref().unwrap()); - } - count if count <= 16 => { - let _pages = Page::alloc_order(1); - page = Some(_pages); - pages = core::slice::from_ref(page.as_ref().unwrap()); - } - count => { - let npages = (count + 15) / 16; - let mut _page_vec = Vec::with_capacity(npages as usize); - for _ in 0..npages { - _page_vec.push(Page::alloc_order(1)); - } - page_vec = Some(_page_vec); - pages = page_vec.as_ref().unwrap().as_slice(); - } - } - - if first_sector_offset != 0 || remaining_data.len() < (sector_count * 512) as usize { - let read_req = BlockDeviceRequest::Read { - sector: sector_start, - count: sector_count, - buffer: pages, - }; - self.commit_request(read_req)?; - } - - let mut data_offset = 0; - let mut page_offset = first_sector_offset as usize; - - for page in pages.iter() { - // SAFETY: We own the page and can modify it - let page_data = unsafe { - let memblk = page.as_memblk(); - core::slice::from_raw_parts_mut(memblk.addr().get() as *mut u8, memblk.len()) - }; + nr_filled += buffer.fill(data)?.allow_partial(); - let copy_len = - (remaining_data.len() - data_offset).min(page_data.len() - page_offset); - - if copy_len == 0 { - break; - } - - page_data[page_offset..page_offset + copy_len] - .copy_from_slice(&remaining_data[data_offset..data_offset + copy_len]); - - data_offset += copy_len; - page_offset = 0; // Only first page has offset - - if data_offset >= remaining_data.len() { + if buffer.available() == 0 { break; } } - let write_req = BlockDeviceRequest::Write { - sector: sector_start, - count: sector_count, - buffer: pages, - }; - self.commit_request(write_req)?; - - let bytes_written = data_offset; - nwritten += bytes_written; - remaining_data = &remaining_data[bytes_written..]; - sector_start += sector_count; - first_sector_offset = 0; + if buffer.available() == 0 { + break; + } } - Ok(nwritten) + if buffer.available() == 0 { + Ok(FillResult::Done(nr_filled)) + } else { + Ok(FillResult::Partial(nr_filled)) + } } } @@ -376,7 +265,7 @@ pub enum BlockDeviceRequest<'lt> { /// Number of sectors to read count: u64, /// Buffer pages to read into - buffer: &'lt [Page], + buffer: &'lt [Folio], }, Write { /// Sector to write to, in 512-byte blocks @@ -384,6 +273,6 @@ pub enum BlockDeviceRequest<'lt> { /// Number of sectors to write count: u64, /// Buffer pages to write from - buffer: &'lt [Page], + buffer: &'lt [Folio], }, } diff --git a/src/kernel/block/mbr.rs b/src/kernel/block/mbr.rs index 74cdc36e..c5820679 100644 --- a/src/kernel/block/mbr.rs +++ b/src/kernel/block/mbr.rs @@ -31,7 +31,7 @@ pub struct MBRPartTable { impl MBRPartTable { pub async fn from_disk(disk: &BlockDevice) -> KResult<Self> { let mut mbr: UninitBuffer<MBRData> = UninitBuffer::new(); - disk.read_some(0, &mut mbr)?.ok_or(EIO)?; + disk.read_some(0, &mut mbr).await?.ok_or(EIO)?; let mbr = mbr.assume_init()?; if mbr.magic != [0x55, 0xaa] { diff --git a/src/kernel/chardev.rs b/src/kernel/chardev.rs index aff3271e..08a21774 100644 --- a/src/kernel/chardev.rs +++ b/src/kernel/chardev.rs @@ -1,23 +1,18 @@ -use super::{ - block::make_device, - console::get_console, - constants::{EEXIST, EIO}, - task::{block_on, ProcessList, Thread}, - terminal::Terminal, - vfs::{DevId, File, FileType, TerminalFile}, -}; -use crate::{ - io::{Buffer, Stream, StreamRead}, - prelude::*, -}; -use alloc::{ - boxed::Box, - collections::btree_map::{BTreeMap, Entry}, - sync::Arc, -}; -use eonix_sync::AsProof as _; +use alloc::boxed::Box; +use alloc::collections::btree_map::{BTreeMap, Entry}; +use alloc::sync::Arc; + use posix_types::open::OpenFlags; +use super::console::get_console; +use super::constants::{EEXIST, EIO}; +use super::task::{block_on, Thread}; +use super::terminal::Terminal; +use super::vfs::types::DeviceId; +use super::vfs::{File, FileType, TerminalFile}; +use crate::io::{Buffer, Stream, StreamRead}; +use crate::prelude::*; + pub trait VirtualCharDevice: Send + Sync { fn read(&self, buffer: &mut dyn Buffer) -> KResult<usize>; fn write(&self, stream: &mut dyn Stream) -> KResult<usize>; @@ -34,12 +29,13 @@ pub struct CharDevice { device: CharDeviceType, } -static CHAR_DEVICES: Spin<BTreeMap<DevId, Arc<CharDevice>>> = Spin::new(BTreeMap::new()); +static CHAR_DEVICES: Spin<BTreeMap<DeviceId, Arc<CharDevice>>> = + Spin::new(BTreeMap::new()); impl CharDevice { - pub fn read(&self, buffer: &mut dyn Buffer) -> KResult<usize> { + pub async fn read(&self, buffer: &mut dyn Buffer) -> KResult<usize> { match &self.device { - CharDeviceType::Terminal(terminal) => block_on(terminal.read(buffer)), + CharDeviceType::Terminal(terminal) => terminal.read(buffer).await, CharDeviceType::Virtual(device) => device.read(buffer), } } @@ -47,18 +43,22 @@ impl CharDevice { pub fn write(&self, stream: &mut dyn Stream) -> KResult<usize> { match &self.device { CharDeviceType::Virtual(device) => device.write(stream), - CharDeviceType::Terminal(terminal) => stream.read_till_end(&mut [0; 128], |data| { - terminal.write(data); - Ok(()) - }), + CharDeviceType::Terminal(terminal) => { + stream.read_till_end(&mut [0; 128], |data| { + terminal.write(data); + Ok(()) + }) + } } } - pub fn get(devid: DevId) -> Option<Arc<CharDevice>> { + pub fn get(devid: DeviceId) -> Option<Arc<CharDevice>> { CHAR_DEVICES.lock().get(&devid).cloned() } - pub fn register(devid: DevId, name: Arc<str>, device: CharDeviceType) -> KResult<()> { + pub fn register( + devid: DeviceId, name: Arc<str>, device: CharDeviceType, + ) -> KResult<()> { match CHAR_DEVICES.lock().entry(devid) { Entry::Vacant(entry) => { entry.insert(Arc::new(CharDevice { name, device })); @@ -68,26 +68,19 @@ impl CharDevice { } } - pub fn open(self: &Arc<Self>, flags: OpenFlags) -> KResult<File> { - Ok(match &self.device { + pub async fn open( + self: &Arc<Self>, thread: &Thread, flags: OpenFlags, + ) -> KResult<File> { + let file = match &self.device { + CharDeviceType::Virtual(_) => { + File::new(flags, FileType::CharDev(self.clone())) + } CharDeviceType::Terminal(terminal) => { - let procs = block_on(ProcessList::get().read()); - let current = Thread::current(); - let session = current.process.session(procs.prove()); - // We only set the control terminal if the process is the session leader. - if session.sid == Thread::current().process.pid { - // Silently fail if we can't set the control terminal. - dont_check!(block_on(session.set_control_terminal( - &terminal, - false, - procs.prove() - ))); - } - - TerminalFile::new(terminal.clone(), flags) + TerminalFile::open(thread, terminal, flags).await } - CharDeviceType::Virtual(_) => File::new(flags, FileType::CharDev(self.clone())), - }) + }; + + Ok(file) } } @@ -134,19 +127,19 @@ impl VirtualCharDevice for ConsoleDevice { impl CharDevice { pub fn init() -> KResult<()> { Self::register( - make_device(1, 3), + DeviceId::new(1, 3), Arc::from("null"), CharDeviceType::Virtual(Box::new(NullDevice)), )?; Self::register( - make_device(1, 5), + DeviceId::new(1, 5), Arc::from("zero"), CharDeviceType::Virtual(Box::new(ZeroDevice)), )?; Self::register( - make_device(5, 1), + DeviceId::new(5, 1), Arc::from("console"), CharDeviceType::Virtual(Box::new(ConsoleDevice)), )?; diff --git a/src/kernel/console.rs b/src/kernel/console.rs index 85fb8b39..3ddef551 100644 --- a/src/kernel/console.rs +++ b/src/kernel/console.rs @@ -1,6 +1,9 @@ -use crate::prelude::*; use alloc::sync::Arc; +use super::constants::EEXIST; +use super::terminal::Terminal; +use crate::prelude::*; + static CONSOLE: Spin<Option<Arc<Terminal>>> = Spin::new(None); pub fn set_console(terminal: Arc<Terminal>) -> KResult<()> { @@ -17,76 +20,3 @@ pub fn get_console() -> Option<Arc<Terminal>> { let console = CONSOLE.lock(); console.clone() } - -#[doc(hidden)] -pub fn _print(args: core::fmt::Arguments) { - // TODO!!!!!!!!!!!!!: REMOVE THIS AND USE `eonix_log`. - eonix_log::do_print(args); -} - -macro_rules! print { - ($($arg:tt)*) => { - $crate::kernel::console::_print(format_args!($($arg)*)) - }; -} - -macro_rules! println { - () => { - $crate::print!("\n") - }; - ($($arg:tt)*) => { - $crate::print!("{}\n", format_args!($($arg)*)) - }; -} - -macro_rules! println_warn { - ($($arg:tt)*) => { - $crate::println!("[kernel: warn] {}", format_args!($($arg)*)) - }; -} - -macro_rules! println_debug { - ($($arg:tt)*) => { - $crate::println!("[kernel:debug] {}", format_args!($($arg)*)) - }; -} - -#[allow(unused_macros)] -macro_rules! println_info { - ($($arg:tt)*) => { - $crate::println!("[kernel: info] {}", format_args!($($arg)*)) - }; -} - -macro_rules! println_fatal { - () => { - $crate::println!("[kernel:fatal] ") - }; - ($($arg:tt)*) => { - $crate::println!("[kernel:fatal] {}", format_args!($($arg)*)) - }; -} - -#[allow(unused_macros)] -macro_rules! println_trace { - ($feat:literal) => { - #[deny(unexpected_cfgs)] - { - #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] ") - } - }; - ($feat:literal, $($arg:tt)*) => {{ - #[deny(unexpected_cfgs)] - { - #[cfg(feature = $feat)] - $crate::println!("[kernel:trace] {}", format_args!($($arg)*)) - } - }}; -} - -use super::{constants::EEXIST, terminal::Terminal}; - -pub(crate) use { - print, println, println_debug, println_fatal, println_info, println_trace, println_warn, -}; diff --git a/src/kernel/constants.rs b/src/kernel/constants.rs index 4e11d66e..b96387b0 100644 --- a/src/kernel/constants.rs +++ b/src/kernel/constants.rs @@ -36,7 +36,7 @@ pub const ENOTDIR: u32 = 20; pub const EISDIR: u32 = 21; pub const EINVAL: u32 = 22; pub const ENOTTY: u32 = 25; -pub const ENOSPC: u32 = 28; +// pub const ENOSPC: u32 = 28; pub const ESPIPE: u32 = 29; // pub const EROFS: u32 = 30; pub const EPIPE: u32 = 32; diff --git a/src/kernel/interrupt.rs b/src/kernel/interrupt.rs index 742727cb..462d2f2c 100644 --- a/src/kernel/interrupt.rs +++ b/src/kernel/interrupt.rs @@ -1,15 +1,17 @@ -use super::mem::handle_kernel_page_fault; -use super::task::block_on; -use super::timer::timer_interrupt; -use crate::kernel::constants::EINVAL; -use crate::prelude::*; use alloc::sync::Arc; + use eonix_hal::traits::fault::Fault; use eonix_hal::traits::trap::{RawTrapContext, TrapType}; use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, VAddr}; use eonix_sync::SpinIrq as _; +use super::mem::handle_kernel_page_fault; +use super::task::block_on; +use super::timer::timer_interrupt; +use crate::kernel::constants::EINVAL; +use crate::prelude::*; + static IRQ_HANDLERS: Spin<[Vec<Arc<dyn Fn() + Send + Sync>>; 16]> = Spin::new([const { Vec::new() }; 16]); @@ -17,7 +19,7 @@ pub fn default_irq_handler(irqno: usize) { assert!(irqno < 16); { - let handlers = IRQ_HANDLERS.lock(); + let handlers = IRQ_HANDLERS.lock_irq(); for handler in handlers[irqno].iter() { handler(); @@ -37,7 +39,9 @@ pub fn default_fault_handler(fault_type: Fault, trap_ctx: &mut TrapContext) { } => { let fault_pc = VAddr::from(trap_ctx.get_program_counter()); - if let Some(new_pc) = block_on(handle_kernel_page_fault(fault_pc, vaddr, error_code)) { + if let Some(new_pc) = + block_on(handle_kernel_page_fault(fault_pc, vaddr, error_code)) + { trap_ctx.set_program_counter(new_pc.addr()); } } @@ -48,7 +52,9 @@ pub fn default_fault_handler(fault_type: Fault, trap_ctx: &mut TrapContext) { #[eonix_hal::default_trap_handler] pub fn interrupt_handler(trap_ctx: &mut TrapContext) { match trap_ctx.trap_type() { - TrapType::Syscall { no, .. } => unreachable!("Syscall {} in kernel space.", no), + TrapType::Syscall { no, .. } => { + unreachable!("Syscall {} in kernel space.", no) + } TrapType::Breakpoint => unreachable!("Breakpoint in kernel space."), TrapType::Fault(fault) => default_fault_handler(fault, trap_ctx), TrapType::Irq { callback } => callback(default_irq_handler), @@ -65,5 +71,12 @@ where } IRQ_HANDLERS.lock_irq()[irqno as usize].push(Arc::new(handler)); + + // On LoongArch the IRQ lines are masked by default and must be unmasked + // once a handler is available, so that an interrupt cannot be delivered + // before anyone can service the device. + #[cfg(target_arch = "loongarch64")] + eonix_hal::arch_exported::interrupt::enable_irq(irqno as usize); + Ok(()) } diff --git a/src/kernel/late_init.rs b/src/kernel/late_init.rs new file mode 100644 index 00000000..9c20be22 --- /dev/null +++ b/src/kernel/late_init.rs @@ -0,0 +1,30 @@ +use alloc::boxed::Box; +use core::future::Future; +use core::pin::Pin; + +use eonix_hal::{extern_symbol_addr, extern_symbol_value}; + +pub fn run_late_init() { + let count = extern_symbol_value!(LATE_INIT_FUNCTION_COUNT); + let addr = extern_symbol_addr!(LATE_INIT_FUNCTIONS, fn()); + + let functions = unsafe { core::slice::from_raw_parts(addr, count) }; + + for func in functions { + func(); + } +} + +pub async fn run_late_init_async() { + let count = extern_symbol_value!(LATE_INIT_ASYNC_FUNCTION_COUNT); + let addr = extern_symbol_addr!( + LATE_INIT_ASYNC_FUNCTIONS, + fn() -> Pin<Box<dyn Future<Output = ()> + Send>> + ); + + let functions = unsafe { core::slice::from_raw_parts(addr, count) }; + + for func in functions { + func().await; + } +} diff --git a/src/kernel/mem.rs b/src/kernel/mem.rs index efd06824..41d5a17d 100644 --- a/src/kernel/mem.rs +++ b/src/kernel/mem.rs @@ -3,14 +3,16 @@ pub mod paging; mod access; mod address; mod allocator; -mod mm_area; +mod folio; mod mm_list; mod page_alloc; mod page_cache; -pub use access::{AsMemoryBlock, MemoryBlock, PhysAccess}; -pub(self) use mm_area::MMArea; -pub use mm_list::{handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission}; +pub use access::PhysAccess; +pub use folio::{Folio, FolioOwned, LockedFolio}; +pub use mm_list::{ + handle_kernel_page_fault, FileMapping, MMList, Mapping, Permission, +}; pub use page_alloc::{GlobalPageAlloc, RawPage}; -pub use page_cache::{CachePage, CachePageStream, PageCache, PageCacheBackend}; -pub use paging::{Page, PageBuffer}; +pub use page_cache::{CachePage, PageCache, PageOffset}; +pub use paging::PageBuffer; diff --git a/src/kernel/mem/access.rs b/src/kernel/mem/access.rs index ce525a0a..328dcfbd 100644 --- a/src/kernel/mem/access.rs +++ b/src/kernel/mem/access.rs @@ -1,22 +1,7 @@ -use core::{num::NonZero, ptr::NonNull}; +use core::ptr::NonNull; use eonix_hal::mm::ArchPhysAccess; use eonix_mm::address::{PAddr, PhysAccess as _PhysAccess}; -/// A block of memory starting at a non-zero address and having a specific length. -/// -/// This struct is used to represent a memory block that can be accessed -/// in the kernel space. -pub struct MemoryBlock { - addr: NonZero<usize>, - len: usize, -} - -pub trait AsMemoryBlock { - /// Translate the physical page the page object pointing to into kernel - /// accessible pointer. Use it with care. - fn as_memblk(&self) -> MemoryBlock; -} - pub trait PhysAccess { /// Translate the data that this address is pointing to into kernel /// accessible pointer. Use it with care. @@ -30,107 +15,6 @@ pub trait PhysAccess { unsafe fn as_ptr<T>(&self) -> NonNull<T>; } -impl MemoryBlock { - /// Create a new `MemoryBlock` with the given address and length. - /// - /// # Safety - /// The caller must ensure that the address is valid. - /// Otherwise, it may lead to undefined behavior. - pub unsafe fn new(addr: NonZero<usize>, len: usize) -> Self { - Self { addr, len } - } - - /// Get the start address of the memory block. - #[allow(dead_code)] - pub fn addr(&self) -> NonZero<usize> { - self.addr - } - - /// Get the length of the memory block. - #[allow(dead_code)] - pub fn len(&self) -> usize { - self.len - } - - /// Split the memory block into two parts at the given offset. - pub fn split_at(&self, at: usize) -> (Self, Self) { - if at > self.len { - panic!("Out of bounds"); - } - - let rhs_start = self.addr.checked_add(at).expect("Overflow"); - - let lhs = unsafe { Self::new(self.addr, at) }; - let rhs = unsafe { Self::new(rhs_start, self.len - at) }; - - (lhs, rhs) - } - - /// Provide a pointer to the data. - /// - /// # Safety - /// Using the returned pointer is undefined behavior if the address is not - /// properly aligned or the size is not equal to the size of `T`. - pub unsafe fn as_ptr_unchecked<T>(&self) -> NonNull<T> { - // SAFETY: `self.addr` is a non-zero value. - NonNull::new_unchecked(self.addr.get() as *mut T) - } - - /// Provide a pointer to the data. - /// - /// # Panic - /// Panic if the address is not properly aligned. - pub fn as_ptr<T>(&self) -> NonNull<T> { - let alignment = align_of::<T>(); - - if self.addr.get() % alignment != 0 { - panic!("Alignment error"); - } - - unsafe { - // SAFETY: We've checked that `self.addr` is properly aligned. - self.as_ptr_unchecked() - } - } - - /// Provide a pointer to the bytes. - pub fn as_byte_ptr(&self) -> NonNull<u8> { - unsafe { - // SAFETY: No alignment check is needed for bytes. - self.as_ptr_unchecked() - } - } - - /// Provide immutable access to the data it pointed to. - /// - /// # Safety - /// This function is unsafe because it returns an immutable reference with - /// a created lifetime. - /// - /// The caller must ensure that the data has no other mutable aliases while - /// the reference is in use. Otherwise, it may lead to undefined behavior. - pub unsafe fn as_bytes<'a>(&self) -> &'a [u8] { - core::slice::from_raw_parts(self.as_ptr_unchecked().as_ptr(), self.len) - } - - /// Provide mutable access to the data it pointed to. - /// - /// # Panic - /// Panic if the address is not properly aligned or the size is not - /// equal to the size of `T`. - /// - /// # Safety - /// This function is unsafe because it returns a mutable reference with a - /// created lifetime. - /// - /// The caller must ensure that the data has no other immutable or mutable - /// aliases while the reference is in use. - /// Otherwise, it may lead to undefined behavior. - pub unsafe fn as_bytes_mut<'a>(&mut self) -> &'a mut [u8] { - core::slice::from_raw_parts_mut(self.as_ptr_unchecked().as_ptr(), self.len) - } -} - impl PhysAccess for PAddr { unsafe fn as_ptr<T>(&self) -> NonNull<T> { ArchPhysAccess::as_ptr(*self) diff --git a/src/kernel/mem/allocator.rs b/src/kernel/mem/allocator.rs index 36b19612..cdcda81e 100644 --- a/src/kernel/mem/allocator.rs +++ b/src/kernel/mem/allocator.rs @@ -1,15 +1,22 @@ -use super::page_alloc::RawPagePtr; -use super::{AsMemoryBlock, GlobalPageAlloc, Page}; +use alloc::format; +use alloc::sync::Arc; use core::alloc::{GlobalAlloc, Layout}; use core::ptr::NonNull; + use eonix_hal::mm::ArchPhysAccess; +use eonix_macros::define_late_init; use eonix_mm::address::PhysAccess; -use eonix_mm::paging::{PAGE_SIZE_BITS, PFN}; +use eonix_mm::paging::{Folio as _, PAGE_SIZE_BITS, PFN}; use eonix_sync::LazyLock; -use slab_allocator::SlabAllocator; +use slab_allocator::SlabAlloc; + +use super::folio::Folio; +use super::GlobalPageAlloc; +use crate::fs::procfs::populate_root; +use crate::io::buf_writeln; -static SLAB_ALLOCATOR: LazyLock<SlabAllocator<RawPagePtr, GlobalPageAlloc, 9>> = - LazyLock::new(|| SlabAllocator::new_in(GlobalPageAlloc)); +static SLAB_ALLOCATOR: LazyLock<SlabAlloc<GlobalPageAlloc, 9>> = + LazyLock::new(|| SlabAlloc::new_in(GlobalPageAlloc)); struct Allocator; @@ -17,37 +24,65 @@ unsafe impl GlobalAlloc for Allocator { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { let size = layout.size().next_power_of_two(); - let result = if size <= 2048 { - SLAB_ALLOCATOR.alloc(size) + if size <= 2048 { + SLAB_ALLOCATOR.alloc(size).as_ptr() } else { - let page_count = size >> PAGE_SIZE_BITS; - let page = Page::alloc_at_least(page_count); - - let ptr = page.as_memblk().as_ptr(); - page.into_raw(); + let folio = Folio::alloc_at_least(size >> PAGE_SIZE_BITS); + let ptr = folio.get_ptr(); + folio.into_raw(); ptr.as_ptr() - }; - - if result.is_null() { - core::ptr::null_mut() - } else { - result as *mut u8 } } unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { let size = layout.size().next_power_of_two(); + let ptr = unsafe { + // SAFETY: The memory we've allocated MUST be non-null. + NonNull::new_unchecked(ptr) + }; if size <= 2048 { SLAB_ALLOCATOR.dealloc(ptr, size) } else { - let paddr = ArchPhysAccess::from_ptr(NonNull::new_unchecked(ptr)); + let paddr = ArchPhysAccess::from_ptr(ptr); let pfn = PFN::from(paddr); - Page::from_raw(pfn); + + Folio::from_raw(pfn); }; } } #[global_allocator] static ALLOCATOR: Allocator = Allocator; + +#[define_late_init] +async fn slabinfo_file() { + populate_root(Arc::from(&b"slabinfo"[..]), |buf| { + let stats = SLAB_ALLOCATOR.dump_stats(); + + buf_writeln!(buf, "slabinfo - version: 2.1")?; + buf_writeln!( + buf, + "# name\t<active_objs>\t<num_objs>\t<objsize>\t<active_slabs>\t<num_slabs>" + )?; + + for (i, stat) in stats.iter().enumerate() { + let name = format!("kmalloc[{i}]"); + + buf_writeln!( + buf, + "{:15}\t{}\t{}\t{}\t{}\t{}", + name, + stat.active_objects, + stat.total_objects, + stat.object_size, + stat.active_slabs, + stat.total_slabs + )?; + } + + Ok(()) + }) + .await; +} diff --git a/src/kernel/mem/folio.rs b/src/kernel/mem/folio.rs new file mode 100644 index 00000000..8ab4d6be --- /dev/null +++ b/src/kernel/mem/folio.rs @@ -0,0 +1,210 @@ +use core::fmt; +use core::mem::ManuallyDrop; +use core::ops::Deref; +use core::ptr::NonNull; +use core::sync::atomic::Ordering; + +use eonix_mm::paging::{Folio as FolioTrait, FrameAlloc, GlobalFrameAlloc, Zone, PFN}; + +use super::page_alloc::ZONE; +use super::{GlobalPageAlloc, PhysAccess as _, RawPage}; + +#[repr(transparent)] +pub struct Folio(NonNull<RawPage>); + +#[derive(Debug)] +#[repr(transparent)] +pub struct FolioOwned(Folio); + +#[repr(transparent)] +pub struct LockedFolio<'a>(&'a Folio); + +unsafe impl Send for Folio {} +unsafe impl Sync for Folio {} + +impl Folio { + pub(super) const fn from_mut_page(raw_page: &'static mut RawPage) -> Self { + Self(NonNull::new(raw_page).unwrap()) + } + + /// Allocate a folio of the given *order*. + pub fn alloc_order(order: u32) -> Self { + GlobalPageAlloc::GLOBAL + .alloc_order(order) + .expect("Out of memory") + } + + /// Allocate a folio of order 0 + pub fn alloc() -> Self { + Self::alloc_order(0) + } + + /// Allocate a folio consisting of at least [`count`] pages. + pub fn alloc_at_least(count: usize) -> Self { + GlobalPageAlloc::GLOBAL + .alloc_at_least(count) + .expect("Out of memory") + } + + /// Acquire the ownership of the folio pointed to by [`pfn`], leaving + /// [`refcount`] untouched. + /// + /// # Panic + /// This function will panic if the folio is not within the global zone. + /// + /// # Safety + /// This function is unsafe because it assumes that the caller has to ensure + /// that [`pfn`] points to a valid folio allocated through [`Self::alloc()`] + /// and that the folio have not been freed or deallocated yet. + pub unsafe fn from_raw(pfn: PFN) -> Self { + unsafe { + // SAFETY: The caller ensures that [`pfn`] points to a folio within + // the global zone. + Self(ZONE.get_page(pfn).unwrap_unchecked()) + } + } + + /// Do some work with the folio without touching the reference count with + /// the same restrictions as [`Self::from_raw()`]. + /// + /// # Safety + /// Check [`Self::from_raw()`] for safety requirements. + pub unsafe fn with_raw<F, O>(pfn: PFN, func: F) -> O + where + F: FnOnce(&Self) -> O, + { + unsafe { + let me = ManuallyDrop::new(Self::from_raw(pfn)); + func(&me) + } + } + + pub fn lock(&self) -> LockedFolio<'_> { + // TODO: actually perform the lock... + LockedFolio(self) + } + + /// Get a vmem pointer to the folio data as a byte slice. + pub fn get_bytes_ptr(&self) -> NonNull<[u8]> { + unsafe { + // SAFETY: `self.start()` can't be null. + NonNull::slice_from_raw_parts(self.start().as_ptr(), self.len()) + } + } + + /// Get a vmem pointer to the start of the folio. + pub fn get_ptr(&self) -> NonNull<u8> { + self.get_bytes_ptr().cast() + } +} + +impl Deref for Folio { + type Target = RawPage; + + fn deref(&self) -> &Self::Target { + unsafe { + // SAFETY: We don't expose mutable references to the folio. + self.0.as_ref() + } + } +} + +impl Clone for Folio { + fn clone(&self) -> Self { + // SAFETY: Memory order here can be Relaxed is for the same reason as + // that in the copy constructor of `std::shared_ptr`. + self.refcount.fetch_add(1, Ordering::Relaxed); + + Self(self.0) + } +} + +impl Drop for Folio { + fn drop(&mut self) { + match self.refcount.fetch_sub(1, Ordering::AcqRel) { + 0 => unreachable!("Refcount for an in-use page is 0"), + 1 => unsafe { GlobalPageAlloc::GLOBAL.dealloc_raw(self.0.as_mut()) }, + _ => {} + } + } +} + +impl fmt::Debug for Folio { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Page({:?}, order={})", self.pfn(), self.order) + } +} + +impl FolioTrait for Folio { + fn pfn(&self) -> PFN { + ZONE.get_pfn(self.0.as_ptr()) + } + + fn order(&self) -> u32 { + self.order + } +} + +impl LockedFolio<'_> { + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts(self.start().as_ptr().as_ptr(), self.len()) + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: `self.start()` points to valid memory of length `self.len()`. + core::slice::from_raw_parts_mut(self.start().as_ptr().as_ptr(), self.len()) + } + } +} + +impl Deref for LockedFolio<'_> { + type Target = Folio; + + fn deref(&self) -> &Self::Target { + self.0 + } +} + +impl FolioOwned { + pub fn alloc() -> Self { + Self(Folio::alloc()) + } + + pub fn alloc_order(order: u32) -> Self { + Self(Folio::alloc_order(order)) + } + + pub fn alloc_at_least(count: usize) -> Self { + Self(Folio::alloc_at_least(count)) + } + + pub fn as_bytes(&self) -> &[u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_ref() + } + } + + pub fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { + // SAFETY: The page is exclusively owned by us. + self.get_bytes_ptr().as_mut() + } + } + + pub fn share(self) -> Folio { + self.0 + } +} + +impl Deref for FolioOwned { + type Target = Folio; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/kernel/mem/mm_area.rs b/src/kernel/mem/mm_area.rs deleted file mode 100644 index 731c5303..00000000 --- a/src/kernel/mem/mm_area.rs +++ /dev/null @@ -1,250 +0,0 @@ -use super::mm_list::EMPTY_PAGE; -use super::paging::AllocZeroed as _; -use super::{AsMemoryBlock, Mapping, Page, Permission}; -use crate::kernel::constants::EINVAL; -use crate::prelude::KResult; -use core::borrow::Borrow; -use core::cell::UnsafeCell; -use core::cmp; -use eonix_mm::address::{AddrOps as _, VAddr, VRange}; -use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; -use eonix_mm::paging::{PAGE_SIZE, PFN}; - -#[derive(Debug)] -pub struct MMArea { - range: UnsafeCell<VRange>, - pub(super) mapping: Mapping, - pub(super) permission: Permission, - pub is_shared: bool, -} - -unsafe impl Send for MMArea {} -unsafe impl Sync for MMArea {} - -impl Clone for MMArea { - fn clone(&self) -> Self { - Self { - range: UnsafeCell::new(self.range()), - mapping: self.mapping.clone(), - permission: self.permission, - is_shared: self.is_shared, - } - } -} - -impl MMArea { - pub fn new(range: VRange, mapping: Mapping, permission: Permission, is_shared: bool) -> Self { - Self { - range: range.into(), - mapping, - permission, - is_shared, - } - } - - fn range_borrow(&self) -> &VRange { - // SAFETY: The only way we get a reference to `MMArea` object is through `MMListInner`. - // And `MMListInner` is locked with IRQ disabled. - unsafe { self.range.get().as_ref().unwrap() } - } - - pub fn range(&self) -> VRange { - *self.range_borrow() - } - - /// # Safety - /// This function should be called only when we can guarantee that the range - /// won't overlap with any other range in some scope. - pub fn grow(&self, count: usize) { - let range = unsafe { self.range.get().as_mut().unwrap() }; - range.clone_from(&self.range_borrow().grow(count)); - } - - pub fn split(mut self, at: VAddr) -> (Option<Self>, Option<Self>) { - assert!(at.is_page_aligned()); - - match self.range_borrow().cmp(&VRange::from(at)) { - cmp::Ordering::Less => (Some(self), None), - cmp::Ordering::Greater => (None, Some(self)), - cmp::Ordering::Equal => { - let diff = at - self.range_borrow().start(); - if diff == 0 { - return (None, Some(self)); - } - - let right = Self { - range: VRange::new(at, self.range_borrow().end()).into(), - permission: self.permission, - mapping: match &self.mapping { - Mapping::Anonymous => Mapping::Anonymous, - Mapping::File(mapping) => Mapping::File(mapping.offset(diff)), - }, - is_shared: self.is_shared, - }; - - let new_range = self.range_borrow().shrink(self.range_borrow().end() - at); - - *self.range.get_mut() = new_range; - (Some(self), Some(right)) - } - } - } - - pub fn handle_cow(&self, pfn: &mut PFN, attr: &mut PageAttribute) { - assert!(attr.contains(PageAttribute::COPY_ON_WRITE)); - - attr.remove(PageAttribute::COPY_ON_WRITE); - attr.set(PageAttribute::WRITE, self.permission.write); - - let page = unsafe { Page::from_raw(*pfn) }; - if page.is_exclusive() { - // SAFETY: This is actually safe. If we read `1` here and we have `MMList` lock - // held, there couldn't be neither other processes sharing the page, nor other - // threads making the page COW at the same time. - core::mem::forget(page); - return; - } - - let new_page; - if *pfn == EMPTY_PAGE.pfn() { - new_page = Page::zeroed(); - } else { - new_page = Page::alloc(); - - unsafe { - // SAFETY: `page` is CoW, which means that others won't write to it. - let old_page_data = page.as_memblk().as_bytes(); - - // SAFETY: `new_page` is exclusive owned by us. - let new_page_data = new_page.as_memblk().as_bytes_mut(); - - new_page_data.copy_from_slice(old_page_data); - }; - } - - attr.remove(PageAttribute::ACCESSED); - *pfn = new_page.into_raw(); - } - - /// # Arguments - /// * `offset`: The offset from the start of the mapping, aligned to 4KB boundary. - pub async fn handle_mmap( - &self, - pfn: &mut PFN, - attr: &mut PageAttribute, - offset: usize, - write: bool, - ) -> KResult<()> { - let Mapping::File(file_mapping) = &self.mapping else { - panic!("Anonymous mapping should not be PA_MMAP"); - }; - - assert!(offset < file_mapping.length, "Offset out of range"); - - let Some(page_cache) = file_mapping.file.page_cache() else { - panic!("Mapping file should have pagecache"); - }; - - let file_offset = file_mapping.offset + offset; - let cnt_to_read = (file_mapping.length - offset).min(0x1000); - - page_cache - .with_page(file_offset, |page, cache_page| { - // Non-write faults: we find page in pagecache and do mapping - // Write fault: we need to care about shared or private mapping. - if !write { - // Bss is embarrassing in pagecache! - // We have to assume cnt_to_read < PAGE_SIZE all bss - if cnt_to_read < PAGE_SIZE { - let new_page = Page::zeroed(); - unsafe { - let page_data = new_page.as_memblk().as_bytes_mut(); - page_data[..cnt_to_read] - .copy_from_slice(&page.as_memblk().as_bytes()[..cnt_to_read]); - } - *pfn = new_page.into_raw(); - } else { - *pfn = page.clone().into_raw(); - } - - if self.permission.write { - if self.is_shared { - // The page may will not be written, - // But we simply assume page will be dirty - cache_page.set_dirty(); - attr.insert(PageAttribute::WRITE); - } else { - attr.insert(PageAttribute::COPY_ON_WRITE); - } - } - } else { - if self.is_shared { - cache_page.set_dirty(); - *pfn = page.clone().into_raw(); - } else { - let new_page = Page::zeroed(); - unsafe { - let page_data = new_page.as_memblk().as_bytes_mut(); - page_data[..cnt_to_read] - .copy_from_slice(&page.as_memblk().as_bytes()[..cnt_to_read]); - } - *pfn = new_page.into_raw(); - } - - attr.insert(PageAttribute::WRITE); - } - }) - .await? - .ok_or(EINVAL)?; - - attr.insert(PageAttribute::PRESENT); - attr.remove(PageAttribute::MAPPED); - Ok(()) - } - - pub async fn handle(&self, pte: &mut impl PTE, offset: usize, write: bool) -> KResult<()> { - let mut attr = pte.get_attr().as_page_attr().expect("Not a page attribute"); - let mut pfn = pte.get_pfn(); - - if attr.contains(PageAttribute::COPY_ON_WRITE) { - self.handle_cow(&mut pfn, &mut attr); - } - - if attr.contains(PageAttribute::MAPPED) { - self.handle_mmap(&mut pfn, &mut attr, offset, write).await?; - } - - attr.insert(PageAttribute::ACCESSED); - - if write { - attr.insert(PageAttribute::DIRTY); - } - - pte.set(pfn, attr.into()); - - Ok(()) - } -} - -impl Eq for MMArea {} -impl PartialEq for MMArea { - fn eq(&self, other: &Self) -> bool { - self.range_borrow().eq(other.range_borrow()) - } -} -impl PartialOrd for MMArea { - fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> { - self.range_borrow().partial_cmp(other.range_borrow()) - } -} -impl Ord for MMArea { - fn cmp(&self, other: &Self) -> core::cmp::Ordering { - self.range_borrow().cmp(other.range_borrow()) - } -} - -impl Borrow<VRange> for MMArea { - fn borrow(&self) -> &VRange { - self.range_borrow() - } -} diff --git a/src/kernel/mem/mm_list.rs b/src/kernel/mem/mm_list.rs index ad1e45c2..c10036ba 100644 --- a/src/kernel/mem/mm_list.rs +++ b/src/kernel/mem/mm_list.rs @@ -1,34 +1,39 @@ +mod brk; mod mapping; +mod mm_area; mod page_fault; +mod page_table; -use super::address::{VAddrExt as _, VRangeExt as _}; -use super::page_alloc::GlobalPageAlloc; -use super::paging::AllocZeroed as _; -use super::{AsMemoryBlock, MMArea, Page}; -use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; -use crate::kernel::mem::page_alloc::RawPagePtr; -use crate::{prelude::*, sync::ArcSwap}; -use alloc::collections::btree_set::BTreeSet; +use alloc::sync::Arc; use core::fmt; use core::sync::atomic::{AtomicUsize, Ordering}; + use eonix_hal::mm::{ - flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, ArchPagingMode, - ArchPhysAccess, GLOBAL_PAGE_TABLE, -}; -use eonix_mm::address::{Addr as _, PAddr}; -use eonix_mm::page_table::PageAttribute; -use eonix_mm::paging::PFN; -use eonix_mm::{ - address::{AddrOps as _, VAddr, VRange}, - page_table::{PageTable, RawAttribute, PTE}, - paging::PAGE_SIZE, + flush_tlb_all, get_root_page_table_pfn, set_root_page_table_pfn, + GLOBAL_PAGE_TABLE, }; +use eonix_mm::address::{Addr as _, AddrOps as _, PAddr, VAddr, VRange}; +use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; +use eonix_mm::paging::{Folio as _, PAGE_SIZE, PFN}; use eonix_sync::{LazyLock, Mutex}; +use mm_area::AreaList; +use page_table::KernelPageTable; -pub use mapping::{FileMapping, Mapping}; -pub use page_fault::handle_kernel_page_fault; +pub use self::mapping::{FileMapping, Mapping}; +pub use self::mm_area::{AreaFlags, MemArea}; +pub use self::page_fault::handle_kernel_page_fault; +use super::address::{VAddrExt as _, VRangeExt as _}; +use super::{Folio, FolioOwned}; +use crate::kernel::constants::{EEXIST, EFAULT, EINVAL, ENOMEM}; +use crate::kernel::mem::mm_list::brk::ProgramBreak; +use crate::prelude::*; +use crate::sync::ArcSwap; -pub static EMPTY_PAGE: LazyLock<Page> = LazyLock::new(|| Page::zeroed()); +pub static EMPTY_PAGE: LazyLock<Folio> = LazyLock::new(|| { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + folio.share() +}); #[derive(Debug, Clone, Copy)] pub struct Permission { @@ -37,37 +42,41 @@ pub struct Permission { pub execute: bool, } -pub type KernelPageTable<'a> = PageTable<'a, ArchPagingMode, GlobalPageAlloc, ArchPhysAccess>; +pub struct MemListLock { + _phantom: (), +} -struct MMListInner<'a> { - areas: BTreeSet<MMArea>, - page_table: KernelPageTable<'a>, - break_start: Option<VRange>, - break_pos: Option<VAddr>, +pub struct MMListInner { + pub lock: MemListLock, + areas: AreaList, + page_table: KernelPageTable, + prog_break: ProgramBreak, } pub struct MMList { - inner: ArcSwap<Mutex<MMListInner<'static>>>, + inner: ArcSwap<Mutex<MMListInner>>, user_count: AtomicUsize, /// Only used in kernel space to switch page tables on context switch. root_page_table: AtomicUsize, } -impl MMListInner<'_> { - fn overlapping_addr(&self, addr: VAddr) -> Option<&MMArea> { - self.areas.get(&VRange::from(addr)) +impl MemListLock { + const fn _new() -> Self { + Self { _phantom: () } } +} - fn check_overlapping_addr(&self, addr: VAddr) -> bool { - addr.is_user() && self.overlapping_addr(addr).is_none() +impl MMListInner { + fn overlapping_addr(&self, addr: VAddr) -> Option<Arc<MemArea>> { + self.areas.get(addr) } - fn overlapping_range(&self, range: VRange) -> impl DoubleEndedIterator<Item = &MMArea> + '_ { - self.areas.range(range.into_bounds()) + fn check_overlapping_addr(&self, addr: VAddr) -> bool { + addr.is_user() && !self.areas.contains(addr) } fn check_overlapping_range(&self, range: VRange) -> bool { - range.is_user() && self.overlapping_range(range).next().is_none() + range.is_user() && !self.areas.contains_range(&range) } fn random_start(&self) -> VAddr { @@ -83,23 +92,28 @@ impl MMListInner<'_> { hint = hint.floor(); } - let mut range = VRange::from(hint).grow(len); - loop { - if !range.is_user() { + let end = hint + len; + + if !VRange::new(hint, end).is_user() { return None; } - match self.overlapping_range(range).next_back() { - None => return Some(range.start()), - Some(area) => { - range = VRange::from(area.range().end().ceil()).grow(len); - } + let Some(ub) = self.areas.upper_bound(end) else { + return Some(hint); + }; + + let ub_range = ub.range.as_ref(&self.lock); + let ub_end = ub_range.end().ceil(); + if ub_end <= hint { + return Some(hint); } + + hint = ub_end; } } - fn unmap(&mut self, start: VAddr, len: usize) -> KResult<Vec<Page>> { + async fn unmap(&mut self, start: VAddr, len: usize) -> KResult<Vec<Folio>> { assert_eq!(start.floor(), start); let end = (start + len).ceil(); let range_to_unmap = VRange::new(start, end); @@ -107,74 +121,30 @@ impl MMListInner<'_> { return Err(EINVAL); } - let mut left_remaining = None; - let mut right_remaining = None; - + // TODO: Write back dirty pages. let mut pages_to_free = Vec::new(); - // TODO: Write back dirty pages. + let isolated_areas = + self.areas.isolate(&range_to_unmap, &mut self.lock).await; - self.areas.retain(|area| { - let Some((left, mid, right)) = area.range().mask_with_checked(&range_to_unmap) else { - return true; - }; + for area in isolated_areas { + let range = area.range.as_ref(&self.lock); - for pte in self.page_table.iter_user(mid) { + for pte in self.page_table.iter_user(*range) { let (pfn, _) = pte.take(); pages_to_free.push(unsafe { // SAFETY: We got the pfn from a valid page table entry, so it should be valid. - Page::from_raw(pfn) + Folio::from_raw(pfn) }); } - - match (left, right) { - (None, None) => {} - (Some(left), None) => { - assert!(left_remaining.is_none()); - let (Some(left), _) = area.clone().split(left.end()) else { - unreachable!("`left.end()` is within the area"); - }; - - left_remaining = Some(left); - } - (None, Some(right)) => { - assert!(right_remaining.is_none()); - let (_, Some(right)) = area.clone().split(right.start()) else { - unreachable!("`right.start()` is within the area"); - }; - - right_remaining = Some(right); - } - (Some(left), Some(right)) => { - assert!(left_remaining.is_none()); - assert!(right_remaining.is_none()); - let (Some(left), Some(mid)) = area.clone().split(left.end()) else { - unreachable!("`left.end()` is within the area"); - }; - - let (_, Some(right)) = mid.split(right.start()) else { - unreachable!("`right.start()` is within the area"); - }; - - left_remaining = Some(left); - right_remaining = Some(right); - } - } - - false - }); - - if let Some(front) = left_remaining { - self.areas.insert(front); - } - if let Some(back) = right_remaining { - self.areas.insert(back); } Ok(pages_to_free) } - fn protect(&mut self, start: VAddr, len: usize, permission: Permission) -> KResult<()> { + async fn protect( + &mut self, start: VAddr, len: usize, permission: Permission, + ) -> KResult<()> { assert_eq!(start.floor(), start); assert!(len != 0); @@ -184,38 +154,22 @@ impl MMListInner<'_> { return Err(EINVAL); } - let mut found = false; - let old_areas = core::mem::take(&mut self.areas); - for mut area in old_areas { - let Some((left, mid, right)) = area.range().mask_with_checked(&range_to_protect) else { - self.areas.insert(area); - continue; - }; + let isolated_areas = + self.areas.isolate(&range_to_protect, &mut self.lock).await; + let mut found = false; + for area in isolated_areas { + let range = area.range.as_ref(&self.lock); found = true; - if let Some(left) = left { - let (Some(left), Some(right)) = area.split(left.end()) else { - unreachable!("`left.end()` is within the area"); - }; - - self.areas.insert(left); - area = right; - } + for pte in self.page_table.iter_user(*range) { + let mut page_attr = pte + .get_attr() + .as_page_attr() + .expect("Not a page attribute"); - if let Some(right) = right { - let (Some(left), Some(right)) = area.split(right.start()) else { - unreachable!("`right.start()` is within the area"); - }; - - self.areas.insert(right); - area = left; - } - - for pte in self.page_table.iter_user(mid) { - let mut page_attr = pte.get_attr().as_page_attr().expect("Not a page attribute"); - - if !permission.read && !permission.write && !permission.execute { + if !permission.read && !permission.write && !permission.execute + { // If no permissions are set, we just remove the page. page_attr.remove( PageAttribute::PRESENT @@ -239,8 +193,10 @@ impl MMListInner<'_> { pte.set_attr(page_attr.into()); } - area.permission = permission; - self.areas.insert(area); + area.set_permission(permission); + + // Insert it back. + self.areas.insert_new(area); } if !found { @@ -251,50 +207,56 @@ impl MMListInner<'_> { } fn mmap( - &mut self, - at: VAddr, - len: usize, - mapping: Mapping, - permission: Permission, - is_shared: bool, + &mut self, at: VAddr, len: usize, mapping: Mapping, + permission: Permission, is_shared: bool, ) -> KResult<()> { assert_eq!(at.floor(), at); assert_eq!(len & (PAGE_SIZE - 1), 0); let range = VRange::new(at, at + len); // We are doing a area marker insertion. - if len == 0 && !self.check_overlapping_addr(at) || !self.check_overlapping_range(range) { + if len == 0 && !self.check_overlapping_addr(at) + || !self.check_overlapping_range(range) + { return Err(EEXIST); } match &mapping { - Mapping::Anonymous => self.page_table.set_anonymous(range, permission), + Mapping::Anonymous => { + self.page_table.set_anonymous(range, permission) + } Mapping::File(_) => self.page_table.set_mmapped(range, permission), } - self.areas - .insert(MMArea::new(range, mapping, permission, is_shared)); + self.areas.insert(MemArea::new( + range, + AreaFlags::from_old(permission, is_shared), + mapping, + )); + Ok(()) } } -impl Drop for MMListInner<'_> { +impl Drop for MMListInner { fn drop(&mut self) { // May buggy - for area in &self.areas { - if area.is_shared { - for pte in self.page_table.iter_user(area.range()) { - let (pfn, _) = pte.take(); - let raw_page = RawPagePtr::from(pfn); - if raw_page.refcount().fetch_sub(1, Ordering::Relaxed) == 1 { - // Wrong here - // unsafe { Page::from_raw(pfn) }; - } + for area in self.areas.iter() { + let range = area.range.as_ref(&self.lock).clone(); + + if area.is_shared() { + for pte in self.page_table.iter_user(range) { + // XXX: Fix me + let _ = pte.take(); + // let raw_page = RawPagePtr::from(pfn); + // if raw_page.refcount().fetch_sub(1, Ordering::Relaxed) == 1 { + // unsafe { Page::from_raw(pfn) }; + // } } } else { - for pte in self.page_table.iter_user(area.range()) { + for pte in self.page_table.iter_user(range) { let (pfn, _) = pte.take(); - unsafe { Page::from_raw(pfn) }; + unsafe { Folio::from_raw(pfn) }; } } } @@ -330,15 +292,15 @@ impl MMList { } pub fn new() -> Self { - let page_table = GLOBAL_PAGE_TABLE.clone_global(); + let page_table = KernelPageTable::new(); Self { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), inner: ArcSwap::new(Mutex::new(MMListInner { - areas: BTreeSet::new(), + lock: MemListLock::_new(), + areas: AreaList::new(), + prog_break: ProgramBreak::null(), page_table, - break_start: None, - break_pos: None, })), } } @@ -347,15 +309,15 @@ impl MMList { let inner = self.inner.borrow(); let mut inner = inner.lock().await; - let page_table = GLOBAL_PAGE_TABLE.clone_global(); + let page_table = KernelPageTable::new(); let list = Self { root_page_table: AtomicUsize::from(page_table.addr().addr()), user_count: AtomicUsize::new(0), inner: ArcSwap::new(Mutex::new(MMListInner { - areas: inner.areas.clone(), + lock: MemListLock::_new(), + areas: inner.areas.deep_clone(&inner.lock), page_table, - break_start: inner.break_start, - break_pos: inner.break_pos, + prog_break: inner.prog_break.clone(), })), }; @@ -363,15 +325,15 @@ impl MMList { let list_inner = list.inner.borrow(); let list_inner = list_inner.lock().await; + let pgtable = &list_inner.page_table; + for area in list_inner.areas.iter() { - if !area.is_shared { - list_inner - .page_table - .set_copy_on_write(&mut inner.page_table, area.range()); + let range = area.range.as_ref(&list_inner.lock).clone(); + + if !area.is_shared() { + pgtable.set_copy_on_write(&mut inner.page_table, range); } else { - list_inner - .page_table - .set_copied(&mut inner.page_table, area.range()); + pgtable.set_copied(&mut inner.page_table, range); } } } @@ -395,26 +357,12 @@ impl MMList { } pub fn deactivate(&self) { - set_root_page_table_pfn(PFN::from(GLOBAL_PAGE_TABLE.addr())); + set_root_page_table_pfn(PFN::from(GLOBAL_PAGE_TABLE.start())); let old_user_count = self.user_count.fetch_sub(1, Ordering::Release); assert_ne!(old_user_count, 0); } - /// Deactivate `self` and activate `to` with root page table changed only once. - /// This might reduce the overhead of switching page tables twice. - #[allow(dead_code)] - pub fn switch(&self, to: &Self) { - self.user_count.fetch_add(1, Ordering::Acquire); - - let root_page_table = self.root_page_table.load(Ordering::Relaxed); - assert_ne!(root_page_table, 0); - set_root_page_table_pfn(PFN::from(PAddr::from(root_page_table))); - - let old_user_count = to.user_count.fetch_sub(1, Ordering::Release); - assert_ne!(old_user_count, 0); - } - /// Replace the current page table with a new one. /// /// # Safety @@ -447,7 +395,7 @@ impl MMList { let new_root_page_table = match &new { Some(new_mm) => new_mm.root_page_table.load(Ordering::Relaxed), - None => GLOBAL_PAGE_TABLE.addr().addr(), + None => GLOBAL_PAGE_TABLE.start().addr(), }; set_root_page_table_pfn(PFN::from(PAddr::from(new_root_page_table))); @@ -457,15 +405,34 @@ impl MMList { // TODO: Check whether we should wake someone up if they've been put // to sleep when calling `vfork`. - self.inner + let old_mm = self + .inner .swap(new.map(|new_mm| new_mm.inner.swap(None)).flatten()); eonix_preempt::enable(); + + // This could take long... + drop(old_mm); + } + + pub fn release(&self) { + let old_mm = self.inner.swap(None); + let old_table = self.root_page_table.swap(0, Ordering::Relaxed); + + // TODO: Remove this completely... + // XXX: `ArcSwap` is broken and never safe to use. Check `replace` above. + assert_ne!(old_table, 0, "Already released?"); + assert!(old_mm.is_some(), "Already released?"); } /// No need to do invalidation manually, `PageTable` already does it. pub async fn unmap(&self, start: VAddr, len: usize) -> KResult<()> { - let pages_to_free = self.inner.borrow().lock().await.unmap(start, len)?; + let pages_to_free = { + let inner = self.inner.borrow(); + let mut inner = inner.lock().await; + + inner.unmap(start, len).await? + }; // We need to assure that the pages are not accessed anymore. // The ones having these pages in their TLB could read from or write to them. @@ -478,8 +445,12 @@ impl MMList { Ok(()) } - pub async fn protect(&self, start: VAddr, len: usize, prot: Permission) -> KResult<()> { - self.inner.borrow().lock().await.protect(start, len, prot)?; + pub async fn protect( + &self, start: VAddr, len: usize, prot: Permission, + ) -> KResult<()> { + let inner = self.inner.borrow(); + let mut inner = inner.lock().await; + inner.protect(start, len, prot).await?; // flush the tlb due to the pte attribute changes self.flush_user_tlbs().await; @@ -529,12 +500,8 @@ impl MMList { } pub async fn mmap_hint( - &self, - hint: VAddr, - len: usize, - mapping: Mapping, - permission: Permission, - is_shared: bool, + &self, hint: VAddr, len: usize, mapping: Mapping, + permission: Permission, is_shared: bool, ) -> KResult<VAddr> { let inner = self.inner.borrow(); let mut inner = inner.lock().await; @@ -557,11 +524,7 @@ impl MMList { } pub async fn mmap_fixed( - &self, - at: VAddr, - len: usize, - mapping: Mapping, - permission: Permission, + &self, at: VAddr, len: usize, mapping: Mapping, permission: Permission, is_shared: bool, ) -> KResult<VAddr> { self.inner @@ -572,77 +535,11 @@ impl MMList { .map(|_| at) } - pub async fn set_break(&self, pos: Option<VAddr>) -> VAddr { - let inner = self.inner.borrow(); - let mut inner = inner.lock().await; - - // SAFETY: `set_break` is only called in syscalls, where program break should be valid. - assert!(inner.break_start.is_some() && inner.break_pos.is_some()); - let break_start = inner.break_start.unwrap(); - let current_break = inner.break_pos.unwrap(); - let pos = match pos { - None => return current_break, - Some(pos) => pos.ceil(), - }; - - if current_break > pos { - return current_break; - } - - let range = VRange::new(current_break, pos); - if !inner.check_overlapping_range(range) { - return current_break; - } - - if !inner.areas.contains(&break_start) { - inner.areas.insert(MMArea::new( - break_start, - Mapping::Anonymous, - Permission { - read: true, - write: true, - execute: false, - }, - false, - )); - } - - let program_break = inner - .areas - .get(&break_start) - .expect("Program break area should be valid"); - - let len = pos - current_break; - let range_to_grow = VRange::from(program_break.range().end()).grow(len); - - program_break.grow(len); - - inner.page_table.set_anonymous( - range_to_grow, - Permission { - read: true, - write: true, - execute: false, - }, - ); - - inner.break_pos = Some(pos); - pos - } - - /// This should be called only **once** for every thread. - pub async fn register_break(&self, start: VAddr) { - let inner = self.inner.borrow(); - let mut inner = inner.lock().await; - assert!(inner.break_start.is_none() && inner.break_pos.is_none()); - - inner.break_start = Some(start.into()); - inner.break_pos = Some(start); - } - /// Access the memory area with the given function. /// The function will be called with the offset of the area and the slice of the area. - pub async fn access_mut<F>(&self, start: VAddr, len: usize, func: F) -> KResult<()> + pub async fn access_mut<F>( + &self, start: VAddr, len: usize, func: F, + ) -> KResult<()> where F: Fn(usize, &mut [u8]), { @@ -662,8 +559,9 @@ impl MMList { while remaining > 0 { let area = inner.overlapping_addr(current).ok_or(EFAULT)?; - let area_start = area.range().start(); - let area_end = area.range().end(); + let range = area.range.as_ref(&inner.lock); + let area_start = range.start(); + let area_end = range.end(); let area_remaining = area_end - current; let access_len = remaining.min(area_remaining); @@ -696,13 +594,12 @@ impl MMList { unsafe { // SAFETY: We are sure that the page is valid and we have the right to access it. - Page::with_raw(pte.get_pfn(), |page| { - // SAFETY: The caller guarantees that no one else is using the page. - let page_data = page.as_memblk().as_bytes_mut(); - func( - offset + idx * 0x1000, - &mut page_data[start_offset..end_offset], - ); + Folio::with_raw(pte.get_pfn(), |page| { + let mut pg = page.lock(); + let page_data = + &mut pg.as_bytes_mut()[start_offset..end_offset]; + + func(offset + idx * 0x1000, page_data); }); } } @@ -729,7 +626,7 @@ trait PageTableExt { fn set_copied(&self, from: &Self, range: VRange); } -impl PageTableExt for KernelPageTable<'_> { +impl PageTableExt for KernelPageTable { fn set_anonymous(&self, range: VRange, permission: Permission) { for pte in self.iter_user(range) { pte.set_anonymous(permission.execute); @@ -789,7 +686,8 @@ where fn set_mapped(&mut self, execute: bool) { // Writable flag is set during page fault handling while executable flag is // preserved across page faults, so we set executable flag now. - let mut attr = PageAttribute::READ | PageAttribute::USER | PageAttribute::MAPPED; + let mut attr = + PageAttribute::READ | PageAttribute::USER | PageAttribute::MAPPED; attr.set(PageAttribute::EXECUTE, execute); self.set(EMPTY_PAGE.clone().into_raw(), T::Attr::from(attr)); @@ -801,7 +699,8 @@ where .as_page_attr() .expect("Not a page attribute"); - if !from_attr.intersects(PageAttribute::PRESENT | PageAttribute::MAPPED) { + if !from_attr.intersects(PageAttribute::PRESENT | PageAttribute::MAPPED) + { return; } @@ -810,7 +709,7 @@ where let pfn = unsafe { // SAFETY: We get the pfn from a valid page table entry, so it should be valid as well. - Page::with_raw(from.get_pfn(), |page| page.clone().into_raw()) + Folio::with_raw(from.get_pfn(), |page| page.clone().into_raw()) }; self.set(pfn, T::Attr::from(from_attr & !PageAttribute::ACCESSED)); diff --git a/src/kernel/mem/mm_list/brk.rs b/src/kernel/mem/mm_list/brk.rs new file mode 100644 index 00000000..5c345b59 --- /dev/null +++ b/src/kernel/mem/mm_list/brk.rs @@ -0,0 +1,159 @@ +use alloc::sync::Arc; + +use eonix_mm::address::{VAddr, VRange}; + +use crate::kernel::mem::address::VRangeExt; +use crate::kernel::mem::mm_list::mm_area::AreaList; +use crate::kernel::mem::mm_list::page_table::KernelPageTable; +use crate::kernel::mem::mm_list::{ + AreaFlags, MemArea, MemListLock, PageTableExt, +}; +use crate::kernel::mem::{MMList, Mapping, Permission}; + +#[derive(Clone)] +pub struct ProgramBreak { + start: VAddr, + current: VAddr, +} + +impl ProgramBreak { + pub const fn new(start: VAddr) -> Self { + Self { + start, + current: start, + } + } + + pub const fn null() -> Self { + Self::new(VAddr::NULL) + } + + pub fn get(&self) -> VAddr { + self.current + } + + pub fn is_null(&self) -> bool { + self.current == VAddr::NULL + } + + fn set(&mut self, pos: VAddr) { + self.current = pos; + } + + fn reset(&mut self, start: VAddr) { + self.start = start; + self.current = start; + } +} + +async fn set_break( + areas: &mut AreaList, brk: &mut ProgramBreak, mm_lock: &mut MemListLock, + page_table: &mut KernelPageTable, newbrk: Option<VAddr>, +) -> VAddr { + assert!( + !brk.is_null(), + "set_break called before program break is registered" + ); + + let curbrk = brk.get(); + let Some(newbrk) = newbrk else { + // Reject NULL brks + return curbrk; + }; + + if newbrk == curbrk { + // Nothing to do + return curbrk; + } + + if newbrk < curbrk { + // TODO: We should allow shrinking the break if there aren't non-brk + // mappings within the range. Reject now to keep things simple. + return curbrk; + } + + let new_range = VRange::new(curbrk, newbrk); + if !new_range.is_user() || areas.contains_range(&new_range) { + return curbrk; + } + + let Some(area) = areas.upper_bound(curbrk) else { + return expand_create_area(areas, brk, page_table, new_range); + }; + + let range = area.range.as_ref(&areas.lock); + + if range.end() != curbrk { + // Someone might have unmapped the brk area. + return expand_create_area(areas, brk, page_table, new_range); + } + + let mut area_lock = area.lock.lock().await; + let range = area.range.as_mut(mm_lock, &mut areas.lock, &mut area_lock); + *range = range.grow(new_range.len()); + + map_break_area(page_table, new_range); + + brk.set(newbrk); + newbrk +} + +fn expand_create_area( + areas: &mut AreaList, brk: &mut ProgramBreak, + page_table: &mut KernelPageTable, new_range: VRange, +) -> VAddr { + let area = Arc::new(MemArea::new( + new_range, + AreaFlags::from_old( + Permission { + read: true, + write: true, + execute: false, + }, + false, + ), + Mapping::Anonymous, + )); + + areas.insert_new(area); + + map_break_area(page_table, new_range); + + brk.set(new_range.end()); + new_range.end() +} + +fn map_break_area(page_table: &mut KernelPageTable, range: VRange) { + page_table.set_anonymous(range, Permission { + read: true, + write: true, + execute: false, + }); +} + +impl MMList { + pub async fn set_break(&self, newbrk: Option<VAddr>) -> VAddr { + let inner = self.inner.borrow(); + let mut inner = inner.lock().await; + let inner = &mut *inner; + + set_break( + &mut inner.areas, + &mut inner.prog_break, + &mut inner.lock, + &mut inner.page_table, + newbrk, + ) + .await + } + + /// This should be called only **once** for every thread. + pub async fn register_break(&self, start: VAddr) { + let inner = self.inner.borrow(); + let mut inner = inner.lock().await; + let brk = &mut inner.prog_break; + + assert!(brk.is_null(), "program break already registered"); + brk.reset(start); + } +} diff --git a/src/kernel/mem/mm_list/mapping.rs b/src/kernel/mem/mm_list/mapping.rs index 662000ba..2b837ae7 100644 --- a/src/kernel/mem/mm_list/mapping.rs +++ b/src/kernel/mem/mm_list/mapping.rs @@ -1,24 +1,18 @@ -use core::fmt::Debug; - -use crate::kernel::vfs::inode::Inode; use alloc::sync::Arc; + use eonix_mm::paging::PAGE_SIZE; +use crate::kernel::mem::PageCache; + #[derive(Debug, Clone)] pub struct FileMapping { - pub file: Arc<dyn Inode>, + pub page_cache: Arc<PageCache>, /// Offset in the file, aligned to 4KB boundary. pub offset: usize, /// Length of the mapping. Exceeding part will be zeroed. pub length: usize, } -impl Debug for dyn Inode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "Inode()") - } -} - #[derive(Debug, Clone)] pub enum Mapping { // private anonymous memory @@ -28,10 +22,10 @@ pub enum Mapping { } impl FileMapping { - pub fn new(file: Arc<dyn Inode>, offset: usize, length: usize) -> Self { + pub fn new(page_cache: Arc<PageCache>, offset: usize, length: usize) -> Self { assert_eq!(offset & (PAGE_SIZE - 1), 0); Self { - file, + page_cache, offset, length, } @@ -39,10 +33,10 @@ impl FileMapping { pub fn offset(&self, offset: usize) -> Self { if self.length <= offset { - Self::new(self.file.clone(), self.offset + self.length, 0) + Self::new(self.page_cache.clone(), self.offset + self.length, 0) } else { Self::new( - self.file.clone(), + self.page_cache.clone(), self.offset + offset, self.length - offset, ) diff --git a/src/kernel/mem/mm_list/mm_area.rs b/src/kernel/mem/mm_list/mm_area.rs new file mode 100644 index 00000000..8892b7d4 --- /dev/null +++ b/src/kernel/mem/mm_list/mm_area.rs @@ -0,0 +1,616 @@ +use alloc::sync::Arc; +use core::cell::UnsafeCell; +use core::mem::ManuallyDrop; +use core::sync::atomic::{AtomicU32, Ordering}; + +use eonix_mm::address::{VAddr, VRange}; +use eonix_mm::page_table::{PageAttribute, RawAttribute, PTE}; +use eonix_mm::paging::{Folio as _, PFN}; +use eonix_sync::Mutex; +use intrusive_collections::rbtree::Entry; +use intrusive_collections::{ + intrusive_adapter, Bound, KeyAdapter, LinkedList, LinkedListAtomicLink, + PointerOps, RBTree, RBTreeAtomicLink, +}; + +use super::{Mapping, EMPTY_PAGE}; +use crate::kernel::mem::folio::Folio; +use crate::kernel::mem::mm_list::MemListLock; +use crate::kernel::mem::{CachePage, FolioOwned, PageOffset, Permission}; +use crate::prelude::KResult; + +bitflags::bitflags! { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub struct AreaFlags: u32 { + const READ = 1 << 0; + const WRITE = 1 << 1; + const EXECUTE = 1 << 2; + const SHARED = 1 << 3; + } +} + +struct AtomicFlags(AtomicU32); + +mod sealed { + pub trait RangeReadLock {} +} + +/// Some lock that can provide read protection for the area range, including a +/// reference to the MemList, its lock, the area list lock and the area lock. +pub trait RangeReadLock: sealed::RangeReadLock {} + +impl RangeReadLock for &MemListLock {} +impl RangeReadLock for &AreaListLock {} +impl RangeReadLock for &AreaLock {} +impl sealed::RangeReadLock for &MemListLock {} +impl sealed::RangeReadLock for &AreaListLock {} +impl sealed::RangeReadLock for &AreaLock {} + +/// # Lock +/// Protected by [`MMListInner`] lock. +pub struct RangeProtected(UnsafeCell<VRange>); + +unsafe impl Send for RangeProtected {} +unsafe impl Sync for RangeProtected {} + +pub struct AreaLock { + _phantom: (), +} + +pub struct AreaListLock { + _phantom: (), +} + +union Link { + rbtree: ManuallyDrop<RBTreeAtomicLink>, + list: ManuallyDrop<LinkedListAtomicLink>, +} + +pub struct MemArea { + /// # Lock + /// Protected by [`MMListInner`] lock. + pub range: RangeProtected, + flags: AtomicFlags, + pub lock: Mutex<AreaLock>, + link: Link, + + mapping: Mapping, +} + +// SAFETY: `areas_link` is larger in size than `LinkedListAtomicLink`. +intrusive_adapter!(ListAdapter = Arc<MemArea>: MemArea { link: LinkedListAtomicLink }); +intrusive_adapter!(AreasAdapter = Arc<MemArea>: MemArea { link: RBTreeAtomicLink }); + +impl<'a> KeyAdapter<'a> for AreasAdapter { + type Key = VRange; + + fn get_key( + &self, value: &'a <Self::PointerOps as PointerOps>::Value, + ) -> Self::Key { + unsafe { + // SAFETY: The [`AreaList`] is within the [`MMListInner`]. + // References to the areas list implies holding the lock of + // [`MMListInner`], so it's safe to read the range. + value.range.as_ref_unchecked().clone() + } + } +} + +pub struct AreaList { + areas: RBTree<AreasAdapter>, + pub lock: AreaListLock, +} + +impl AreaLock { + const fn _new() -> Self { + Self { _phantom: () } + } +} + +impl AreaFlags { + const RWX: Self = Self::READ.union(Self::WRITE).union(Self::EXECUTE); + + pub fn from_old(perm: Permission, is_shared: bool) -> Self { + let mut flags = AreaFlags::empty(); + + flags.set(AreaFlags::READ, perm.read); + flags.set(AreaFlags::WRITE, perm.write); + flags.set(AreaFlags::EXECUTE, perm.execute); + flags.set(AreaFlags::SHARED, is_shared); + + flags + } +} + +impl AtomicFlags { + const fn new(flags: AreaFlags) -> Self { + Self(AtomicU32::new(flags.bits())) + } + + /// Load with Acquire semantics. + fn load(&self) -> AreaFlags { + AreaFlags::from_bits_retain(self.0.load(Ordering::Acquire)) + } + + /// Do atomic read-modify-write with AcqRel semantics on success and + /// Relaxed semantics on failure. + fn update( + &self, mut try_modify: impl FnMut(AreaFlags) -> AreaFlags, + ) -> AreaFlags { + loop { + let Ok(old_raw) = self.0.fetch_update( + Ordering::AcqRel, + Ordering::Relaxed, + |raw| { + let flags = AreaFlags::from_bits_retain(raw); + Some(try_modify(flags).bits()) + }, + ) else { + continue; + }; + + return AreaFlags::from_bits_retain(old_raw); + } + } +} + +impl Clone for AtomicFlags { + fn clone(&self) -> Self { + Self::new(self.load()) + } +} + +impl RangeProtected { + pub const fn new(range: VRange) -> Self { + Self(UnsafeCell::new(range)) + } + + unsafe fn as_ref_unchecked<'a>(&self) -> &'a VRange { + unsafe { &*self.0.get() } + } + + pub fn as_ref<'a>(&self, _lock: impl RangeReadLock + 'a) -> &'a VRange { + unsafe { + // SAFETY: We are holding some kind of read safety lock. + self.as_ref_unchecked() + } + } + + pub fn as_mut<'a>( + &self, _mm_list_lock: &'a mut MemListLock, + _list_write_lock: &'a mut AreaListLock, _area_lock: &'a mut AreaLock, + ) -> &'a mut VRange { + unsafe { + // SAFETY: If we are holding the list's write lock, we can guarantee + // that no one else is reading or writing the areas, so it's safe to + // return a mutable reference. + &mut *self.0.get() + } + } +} + +impl AreaList { + pub const fn new() -> Self { + Self { + areas: RBTree::new(AreasAdapter::NEW), + lock: AreaListLock { _phantom: () }, + } + } + + pub fn insert_new(&mut self, area: Arc<MemArea>) { + let range = area.range.as_ref(&self.lock).clone(); + + match self.areas.entry(&range) { + Entry::Occupied(_) => panic!("Overlapping mem area: {range:?}."), + Entry::Vacant(insert_cursor) => { + insert_cursor.insert(area); + } + } + } + + pub fn get(&self, addr: VAddr) -> Option<Arc<MemArea>> { + self.areas.find(&VRange::from(addr)).clone_pointer() + } + + pub fn contains(&self, addr: VAddr) -> bool { + !self.areas.find(&VRange::from(addr)).is_null() + } + + /// Return the mem area with the highest start address that is below or + /// overlaps with the given address. + pub fn upper_bound(&self, addr: VAddr) -> Option<Arc<MemArea>> { + self.areas + .upper_bound(Bound::Included(&VRange::from(addr))) + .clone_pointer() + } + + pub fn contains_range(&self, range: &VRange) -> bool { + let Some(ub) = self.upper_bound(range.end()) else { + return false; + }; + + let range = ub.range.as_ref(&self.lock); + range.end() <= range.start() + } + + /// Isolate the given range by splitting the areas that overlap with it, and + /// return the areas that are fully covered by the range and removed. + pub async fn isolate( + &mut self, isolate_range: &VRange, lock: &mut MemListLock, + ) -> impl Iterator<Item = Arc<MemArea>> { + struct SendFuture<F>(F); + unsafe impl<F> Send for SendFuture<F> {} + unsafe impl<F> Sync for SendFuture<F> {} + impl<F> core::future::Future for SendFuture<F> + where + F: core::future::Future, + { + type Output = F::Output; + + fn poll( + self: core::pin::Pin<&mut Self>, + cx: &mut core::task::Context<'_>, + ) -> core::task::Poll<Self::Output> { + unsafe { self.map_unchecked_mut(|s| &mut s.0).poll(cx) } + } + } + + SendFuture(self.isolate_(isolate_range, lock)).await + } + + async fn isolate_( + &mut self, isolate_range: &VRange, lock: &mut MemListLock, + ) -> impl Iterator<Item = Arc<MemArea>> { + let mut ret_areas = LinkedList::new(ListAdapter::NEW); + let begin = VRange::from(isolate_range.start()); + + let mut cursor = self.areas.upper_bound_mut(Bound::Included(&begin)); + + while !cursor.is_null() { + let range = { + let area = cursor.get().unwrap(); + let range = area.range.as_ref(&*lock); + + if range.start() >= isolate_range.end() { + break; + } + + if range.end() <= isolate_range.start() { + cursor.move_next(); + continue; + } + + range.clone() + }; + + let area = cursor.as_cursor().clone_pointer().unwrap(); + let list_lock = &mut self.lock; + let mut area_lock = area.lock.lock().await; + let area_lock = &mut area_lock; + + let (l, m, r) = range.mask_with_checked(isolate_range).unwrap(); + + match (l, r) { + (None, None) => { + // Fully covered, just remove it. + let area = cursor.remove().unwrap(); + unsafe { + area.link.to_list(); + } + ret_areas.push_back(area); + continue; + } + (None, Some(rem)) | (Some(rem), None) => { + // Overflow on one side, change the old area's range and + // return the newly created area. + let range = area.range.as_mut(lock, list_lock, area_lock); + *range = rem; + } + (Some(left), Some(right)) => { + // Overflow on both sides, change the old area's range to + // the left part, create a new area for the right part and + // return the middle part. + let range = area.range.as_mut(lock, list_lock, area_lock); + *range = left; + + cursor.insert_after(area.clone_and_modify(lock, |area| { + area.range = RangeProtected::new(right); + })); + } + } + + let ret_range = area.clone_and_modify(lock, |area| { + area.range = RangeProtected::new(m); + area.link = Link::list(); + }); + + ret_areas.push_back(ret_range); + } + + // We are returning the area, and the area should be by default + // in rbtrees, so we need to convert the link back to rbtree + // link so the caller won't be surprised. + ret_areas.into_iter().inspect(|area| unsafe { + // SAFETY: `into_iter()` removes the area from the list. + area.link.to_rbtree() + }) + } + + // TODO: For backwards compatibility. Remove this. + pub fn insert(&mut self, area: MemArea) { + let range = area.range.as_ref(&self.lock).clone(); + + match self.areas.entry(&range) { + Entry::Occupied(_) => panic!("Overlapping mem area: {:?}.", range), + Entry::Vacant(insert_cursor) => { + insert_cursor.insert(Arc::new(area)); + } + } + } + + // TODO: For backwards compatibility. Remove this. + pub fn deep_clone(&self, lock: &MemListLock) -> Self { + let mut areas = RBTree::new(AreasAdapter::NEW); + + for area in self.areas.iter() { + areas.insert(Arc::new(area.clone(lock))); + } + + Self { + areas, + lock: AreaListLock { _phantom: () }, + } + } + + pub fn iter(&self) -> impl Iterator<Item = &MemArea> { + let mut cursor = self.areas.front(); + + core::iter::from_fn(move || { + let area = cursor.get()?; + cursor.move_next(); + + Some(area) + }) + } +} + +impl MemArea { + pub const fn new( + range: VRange, flags: AreaFlags, mapping: Mapping, + ) -> Self { + Self { + range: RangeProtected::new(range), + flags: AtomicFlags::new(flags), + lock: Mutex::new(AreaLock::_new()), + link: Link::rbtree(), + mapping, + } + } + + pub fn set_permission(&self, perm: Permission) { + let mut new_flags = AreaFlags::empty(); + + if perm.read { + new_flags.insert(AreaFlags::READ); + } + + if perm.write { + new_flags.insert(AreaFlags::WRITE); + } + + if perm.execute { + new_flags.insert(AreaFlags::EXECUTE); + } + + self.flags + .update(|flags| (flags & !AreaFlags::RWX) | new_flags); + } + + pub fn is_shared(&self) -> bool { + self.flags.load().contains(AreaFlags::SHARED) + } + + pub fn can_read(&self) -> bool { + self.flags.load().contains(AreaFlags::READ) + } + + pub fn can_write(&self) -> bool { + self.flags.load().contains(AreaFlags::WRITE) + } + + pub fn can_execute(&self) -> bool { + self.flags.load().contains(AreaFlags::EXECUTE) + } + + pub fn clone(&self, lock: &MemListLock) -> Self { + Self { + range: RangeProtected::new(self.range.as_ref(lock).clone()), + flags: self.flags.clone(), + lock: Mutex::new(AreaLock::_new()), + link: Link::rbtree(), + mapping: self.mapping.clone(), + } + } + + fn clone_and_modify( + &self, lock: &MemListLock, modify: impl FnOnce(&mut MemArea), + ) -> Arc<Self> { + let mut arc = Arc::new(self.clone(lock)); + let arc_mut = unsafe { + // SAFETY: We are the only owner. + Arc::get_mut(&mut arc).unwrap_unchecked() + }; + modify(arc_mut); + arc + } + + pub fn handle_cow(&self, pfn: &mut PFN, attr: &mut PageAttribute) { + assert!(attr.contains(PageAttribute::COPY_ON_WRITE)); + + attr.remove(PageAttribute::COPY_ON_WRITE); + attr.set(PageAttribute::WRITE, self.can_write()); + + let page = unsafe { Folio::from_raw(*pfn) }; + + // XXX: Change me!!! + if page.refcount.load(Ordering::Relaxed) == 1 { + // SAFETY: This is actually safe. If we read `1` here and we have `MMList` lock + // held, there couldn't be neither other processes sharing the page, nor other + // threads making the page COW at the same time. + core::mem::forget(page); + return; + } + + let mut new_page; + if *pfn == EMPTY_PAGE.pfn() { + new_page = { + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); + folio + }; + } else { + new_page = FolioOwned::alloc(); + + unsafe { + // SAFETY: `page` is CoW, which means that others won't write to it. + let old_page_data = page.get_bytes_ptr().as_ref(); + let new_page_data = new_page.as_bytes_mut(); + + new_page_data.copy_from_slice(old_page_data); + }; + } + + attr.remove(PageAttribute::ACCESSED); + *pfn = new_page.share().into_raw(); + } + + /// # Arguments + /// * `offset`: The offset from the start of the mapping, aligned to 4KB boundary. + pub async fn handle_mmap( + &self, pfn: &mut PFN, attr: &mut PageAttribute, offset: usize, + write: bool, + ) -> KResult<()> { + let Mapping::File(file_mapping) = &self.mapping else { + panic!("Anonymous mapping should not be PA_MMAP"); + }; + + assert!(offset < file_mapping.length, "Offset out of range"); + + let file_offset = file_mapping.offset + offset; + + let map_page = |cache_page: &CachePage| { + if !self.can_write() { + assert!(!write, "Write fault on read-only mapping"); + + *pfn = cache_page.add_mapping(); + return; + } + + if self.is_shared() { + // We don't process dirty flags in write faults. + // Simply assume that page will eventually be dirtied. + // So here we can set the dirty flag now. + cache_page.set_dirty(true); + attr.insert(PageAttribute::WRITE); + *pfn = cache_page.add_mapping(); + return; + } + + if !write { + // Delay the copy-on-write until write fault happens. + attr.insert(PageAttribute::COPY_ON_WRITE); + *pfn = cache_page.add_mapping(); + return; + } + + // XXX: Change this. Let's handle mapped pages before CoW pages. + // Nah, we are writing to a mapped private mapping... + let mut new_page = FolioOwned::alloc(); + new_page + .as_bytes_mut() + .copy_from_slice(cache_page.lock().as_bytes()); + + attr.insert(PageAttribute::WRITE); + *pfn = new_page.share().into_raw(); + }; + + file_mapping + .page_cache + .with_page(PageOffset::from_byte_floor(file_offset), map_page) + .await?; + + attr.insert(PageAttribute::PRESENT); + attr.remove(PageAttribute::MAPPED); + Ok(()) + } + + pub async fn handle( + &self, pte: &mut impl PTE, offset: usize, write: bool, + ) -> KResult<()> { + // Exclude concurrent modifications and faults. + // TODO: concurrent faults should be acceptable... + let lock = self.lock.lock().await; + assert!( + offset < self.range.as_ref(&*lock).len(), + "Offset out of range" + ); + + let mut attr = + pte.get_attr().as_page_attr().expect("Not a page attribute"); + let mut pfn = pte.get_pfn(); + + if attr.contains(PageAttribute::COPY_ON_WRITE) { + self.handle_cow(&mut pfn, &mut attr); + } + + if attr.contains(PageAttribute::MAPPED) { + self.handle_mmap(&mut pfn, &mut attr, offset, write).await?; + } + + attr.insert(PageAttribute::ACCESSED); + + if write { + attr.insert(PageAttribute::DIRTY); + } + + pte.set(pfn, attr.into()); + + Ok(()) + } +} + +impl Link { + const fn rbtree() -> Self { + Self { + rbtree: ManuallyDrop::new(RBTreeAtomicLink::new()), + } + } + + const fn list() -> Self { + Self { + list: ManuallyDrop::new(LinkedListAtomicLink::new()), + } + } + + /// Convert from RBTree link to LinkedList link. + /// + /// # Safety + /// The caller must guarantee that the link is currently not linked. + /// Otherwise, this is an undefined behavior. + unsafe fn to_list(&self) { + unsafe { + self.list.force_unlink(); + } + } + + /// Convert from LinkedList link to RBTree link. + /// + /// # Safety + /// The caller must guarantee that the link is currently not linked. + /// Otherwise, this is an undefined behavior. + unsafe fn to_rbtree(&self) { + unsafe { + self.rbtree.force_unlink(); + } + } +} diff --git a/src/kernel/mem/mm_list/page_fault.rs b/src/kernel/mem/mm_list/page_fault.rs index 6f14583d..28c2c5ef 100644 --- a/src/kernel/mem/mm_list/page_fault.rs +++ b/src/kernel/mem/mm_list/page_fault.rs @@ -1,11 +1,13 @@ -use super::{MMList, VAddr}; -use crate::kernel::task::Thread; +use eonix_hal::extern_symbol_addr; use eonix_hal::mm::flush_tlb; use eonix_hal::traits::fault::PageFaultErrorCode; use eonix_mm::address::{Addr as _, AddrOps as _, VRange}; use eonix_mm::paging::PAGE_SIZE; use posix_types::signal::Signal; +use super::{MMList, VAddr}; +use crate::kernel::task::Thread; + #[repr(C)] struct FixEntry { start: u64, @@ -23,27 +25,20 @@ impl FixEntry { VAddr::from((self.start + self.length) as usize) } - #[allow(dead_code)] - fn range(&self) -> VRange { - VRange::new(self.start(), self.end()) - } - fn jump_address(&self) -> VAddr { VAddr::from(self.jump_address as usize) } fn entries() -> &'static [FixEntry] { - extern "C" { - fn FIX_START(); - fn FIX_END(); - } + let fix_seg_len_bytes = + extern_symbol_addr!(FIX_END) - extern_symbol_addr!(FIX_START); unsafe { - // SAFETY: `FIX_START` and `FIX_END` are defined in the - // linker script in `.rodata` section. + // SAFETY: `FIX_START` and `FIX_END` are defined in the linker script + // in `.rodata` section. core::slice::from_raw_parts( - FIX_START as usize as *const FixEntry, - (FIX_END as usize - FIX_START as usize) / size_of::<FixEntry>(), + extern_symbol_addr!(FIX_START, FixEntry), + fix_seg_len_bytes / size_of::<FixEntry>(), ) } } @@ -52,9 +47,7 @@ impl FixEntry { impl MMList { /// Handle a user page fault. pub async fn handle_user_page_fault( - &self, - addr: VAddr, - error: PageFaultErrorCode, + &self, addr: VAddr, error: PageFaultErrorCode, ) -> Result<(), Signal> { debug_assert!( error.contains(PageFaultErrorCode::UserAccess), @@ -64,10 +57,10 @@ impl MMList { let inner = self.inner.borrow(); let inner = inner.lock().await; - let area = inner.areas.get(&VRange::from(addr)).ok_or(Signal::SIGBUS)?; + let area = inner.areas.get(addr).ok_or(Signal::SIGBUS)?; // Check user access permission. - if error.contains(PageFaultErrorCode::Read) && !area.permission.read { + if error.contains(PageFaultErrorCode::Read) && !area.can_read() { // Under x86_64, we don't have a way to distinguish // between a read fault and a non-present fault. But it should be OK // since non-readable pages are not allowed under x86 and if we read @@ -75,11 +68,13 @@ impl MMList { Err(Signal::SIGSEGV)? } - if error.contains(PageFaultErrorCode::Write) && !area.permission.write { + if error.contains(PageFaultErrorCode::Write) && !area.can_write() { Err(Signal::SIGSEGV)? } - if error.contains(PageFaultErrorCode::InstructionFetch) && !area.permission.execute { + if error.contains(PageFaultErrorCode::InstructionFetch) + && !area.can_execute() + { Err(Signal::SIGSEGV)? } @@ -91,7 +86,7 @@ impl MMList { area.handle( pte, - addr.floor() - area.range().start(), + addr.floor() - area.range.as_ref(&inner.lock).start(), error.contains(PageFaultErrorCode::Write), ) .await @@ -130,9 +125,7 @@ fn kernel_page_fault_die(vaddr: VAddr, pc: VAddr) -> ! { } pub async fn handle_kernel_page_fault( - fault_pc: VAddr, - addr: VAddr, - error: PageFaultErrorCode, + fault_pc: VAddr, addr: VAddr, error: PageFaultErrorCode, ) -> Option<VAddr> { debug_assert!( !error.contains(PageFaultErrorCode::UserAccess), @@ -151,23 +144,22 @@ pub async fn handle_kernel_page_fault( let inner = mms.inner.borrow(); let inner = inner.lock().await; - let area = match inner.areas.get(&VRange::from(addr)) { - Some(area) => area, - None => { - return Some(try_page_fault_fix(fault_pc, addr)); - } + let Some(area) = inner.areas.get(addr) else { + return Some(try_page_fault_fix(fault_pc, addr)); }; let pte = inner .page_table .iter_user(VRange::from(addr.floor()).grow(PAGE_SIZE)) .next() - .expect("If we can find the mapped area, we should be able to find the PTE"); + .expect( + "If we can find the mapped area, we should be able to find the PTE", + ); if let Err(_) = area .handle( pte, - addr.floor() - area.range().start(), + addr.floor() - area.range.as_ref(&inner.lock).start(), error.contains(PageFaultErrorCode::Write), ) .await diff --git a/src/kernel/mem/mm_list/page_table.rs b/src/kernel/mem/mm_list/page_table.rs new file mode 100644 index 00000000..8a2acc13 --- /dev/null +++ b/src/kernel/mem/mm_list/page_table.rs @@ -0,0 +1,40 @@ +use core::ops::Deref; + +use eonix_hal::arch_exported::mm::{ArchPagingMode, PageAccessImpl}; +use eonix_hal::mm::GLOBAL_PAGE_TABLE; +use eonix_mm::page_table::PageTable; +use eonix_mm::paging::{Folio, GlobalFrameAlloc}; + +use crate::kernel::mem::{FolioOwned, GlobalPageAlloc, PhysAccess}; + +#[repr(transparent)] +pub struct KernelPageTable(PageTable<'static, ArchPagingMode, GlobalPageAlloc, PageAccessImpl>); + +impl KernelPageTable { + pub fn new() -> Self { + let global_page_table = unsafe { + // SAFETY: The region is valid and read only after initialization. + GLOBAL_PAGE_TABLE.start().as_ptr::<[u8; 4096]>().as_ref() + }; + + let mut table_page = FolioOwned::alloc(); + let entries = table_page.as_bytes_mut().len(); + table_page.as_bytes_mut()[..(entries / 2)].fill(0); + table_page.as_bytes_mut()[(entries / 2)..] + .copy_from_slice(&global_page_table[(entries / 2)..]); + + Self(PageTable::new( + table_page.share(), + GlobalPageAlloc::GLOBAL, + PageAccessImpl, + )) + } +} + +impl Deref for KernelPageTable { + type Target = PageTable<'static, ArchPagingMode, GlobalPageAlloc, PageAccessImpl>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/kernel/mem/page_alloc.rs b/src/kernel/mem/page_alloc.rs index fcbe9bb3..f60332d7 100644 --- a/src/kernel/mem/page_alloc.rs +++ b/src/kernel/mem/page_alloc.rs @@ -1,21 +1,31 @@ mod raw_page; +mod zones; -use buddy_allocator::{BuddyAllocator, BuddyRawPage as _}; +use alloc::sync::Arc; use core::sync::atomic::Ordering; -use eonix_mm::{ - address::{AddrOps as _, PRange}, - paging::{GlobalPageAlloc as GlobalPageAllocTrait, PageAlloc, PFN}, + +use buddy_allocator::BuddyAllocator; +use eonix_macros::define_late_init; +use eonix_mm::address::PRange; +use eonix_mm::page_table::PageTableAlloc; +use eonix_mm::paging::{ + FolioList, FolioListSized as _, FrameAlloc, GlobalFrameAlloc, PFN, }; +use eonix_preempt::PreemptGuard; use eonix_sync::{NoContext, Spin}; -use intrusive_list::List; -use raw_page::PageFlags; +pub use raw_page::{PageFlags, RawPage, RawPageList}; +pub use zones::{GlobalZone, ZONE}; -pub use raw_page::{RawPage, RawPagePtr}; +use super::folio::Folio; +use crate::fs::procfs::populate_root; +use crate::io::buf_write; const COSTLY_ORDER: u32 = 3; +const AREAS: usize = COSTLY_ORDER as usize + 1; const BATCH_SIZE: u32 = 64; -static BUDDY_ALLOC: Spin<BuddyAllocator<RawPagePtr>> = Spin::new(BuddyAllocator::new()); +static BUDDY_ALLOC: Spin<BuddyAllocator<GlobalZone, RawPageList>> = + Spin::new(BuddyAllocator::new(&GlobalZone())); #[eonix_percpu::define_percpu] static PERCPU_PAGE_ALLOC: PerCpuPageAlloc = PerCpuPageAlloc::new(); @@ -23,82 +33,48 @@ static PERCPU_PAGE_ALLOC: PerCpuPageAlloc = PerCpuPageAlloc::new(); #[derive(Clone)] pub struct GlobalPageAlloc; -#[derive(Clone)] -pub struct BuddyPageAlloc(); - struct PerCpuPageAlloc { batch: u32, - // TODO: might be used in the future. - // high: u32, - free_areas: [List; COSTLY_ORDER as usize + 1], + free_areas: [RawPageList; AREAS], +} + +pub trait PerCpuPage { + fn set_local(&mut self, val: bool); } impl PerCpuPageAlloc { const fn new() -> Self { Self { batch: BATCH_SIZE, - // high: 0, - free_areas: [const { List::new() }; COSTLY_ORDER as usize + 1], + free_areas: [RawPageList::NEW; AREAS], } } - fn insert_free_pages(&mut self, pages_ptr: RawPagePtr, order: u32) { - let free_area = &mut self.free_areas[order as usize]; - free_area.insert(unsafe { pages_ptr.get_link() }); - } - - fn get_free_pages(&mut self, order: u32) -> Option<RawPagePtr> { - let free_area = &mut self.free_areas[order as usize]; - free_area.pop().map(|node| unsafe { - // SAFETY: `node` is a valid pointer to a `Link` that is not used by anyone. - RawPagePtr::from_link(node) - }) - } - - fn alloc_order(&mut self, order: u32) -> Option<RawPagePtr> { + fn alloc_order(&mut self, order: u32) -> Option<&'static mut RawPage> { assert!(order <= COSTLY_ORDER); - if let Some(pages) = self.get_free_pages(order) { + if let Some(pages) = self.free_areas[order as usize].pop_head() { return Some(pages); } let batch = self.batch >> order; for _ in 0..batch { - if let Some(pages_ptr) = BUDDY_ALLOC.lock().alloc_order(order) { - pages_ptr.flags().set(PageFlags::LOCAL); - self.insert_free_pages(pages_ptr, order); - } else { + let Some(page) = BUDDY_ALLOC.lock().alloc_order(order) else { break; }; + + page.set_local(true); + self.free_areas[order as usize].push_tail(page); } - self.get_free_pages(order) + self.free_areas[order as usize].pop_head() } - fn free_pages(&mut self, pages_ptr: RawPagePtr, order: u32) { - assert_eq!(pages_ptr.order(), order); - assert_eq!(pages_ptr.refcount().load(Ordering::Relaxed), 0); - - pages_ptr.refcount().store(1, Ordering::Relaxed); - self.insert_free_pages(pages_ptr, order); + fn free_pages(&mut self, page: &'static mut RawPage, order: u32) { + self.free_areas[order as usize].push_tail(page); } } impl GlobalPageAlloc { - #[allow(dead_code)] - pub const fn buddy_alloc() -> BuddyPageAlloc { - BuddyPageAlloc() - } - - pub fn mark_present(range: PRange) { - let mut pfn = PFN::from(range.start().ceil()); - let end_pfn = PFN::from(range.end().floor()); - - while pfn < end_pfn { - RawPagePtr::from(pfn).flags().set(PageFlags::PRESENT); - pfn = pfn + 1; - } - } - /// Add the pages in the PAddr range `range` to the global allocator. /// /// This function is only to be called on system initialization when `eonix_preempt` @@ -110,63 +86,93 @@ impl GlobalPageAlloc { pub unsafe fn add_pages(range: PRange) { BUDDY_ALLOC .lock_with_context(NoContext) - .create_pages(range.start(), range.end()) + .create_folios(range.start(), range.end()) } -} - -impl PageAlloc for GlobalPageAlloc { - type RawPage = RawPagePtr; - fn alloc_order(&self, order: u32) -> Option<RawPagePtr> { + pub fn alloc_raw_order(&self, order: u32) -> Option<&'static mut RawPage> { if order > COSTLY_ORDER { BUDDY_ALLOC.lock().alloc_order(order) } else { unsafe { eonix_preempt::disable(); - let page_ptr = PERCPU_PAGE_ALLOC.as_mut().alloc_order(order); + let page = PERCPU_PAGE_ALLOC.as_mut().alloc_order(order); eonix_preempt::enable(); - page_ptr + + page } } } - unsafe fn dealloc(&self, page_ptr: RawPagePtr) { - if page_ptr.order() > COSTLY_ORDER { - BUDDY_ALLOC.lock().dealloc(page_ptr); + pub unsafe fn dealloc_raw(&self, raw_page: &'static mut RawPage) { + assert_eq!( + raw_page.refcount.load(Ordering::Relaxed), + 0, + "Trying to free a page with refcount > 0" + ); + + if raw_page.order > COSTLY_ORDER { + BUDDY_ALLOC.lock().dealloc(raw_page); } else { - let order = page_ptr.order(); + let order = raw_page.order; + unsafe { - eonix_preempt::disable(); - PERCPU_PAGE_ALLOC.as_mut().free_pages(page_ptr, order); - eonix_preempt::enable(); + PreemptGuard::new(PERCPU_PAGE_ALLOC.as_mut()) + .free_pages(raw_page, order); } } } +} + +impl FrameAlloc for GlobalPageAlloc { + type Folio = Folio; + + fn alloc_order(&self, order: u32) -> Option<Self::Folio> { + self.alloc_raw_order(order).map(|raw_page| { + // SAFETY: Memory order here can be Relaxed is for the same reason + // as that in the copy constructor of `std::shared_ptr`. - fn has_management_over(&self, page_ptr: RawPagePtr) -> bool { - BuddyAllocator::has_management_over(page_ptr) - && (page_ptr.order() > COSTLY_ORDER || page_ptr.flags().has(PageFlags::LOCAL)) + raw_page.refcount.fetch_add(1, Ordering::Relaxed); + Folio::from_mut_page(raw_page) + }) } } -impl GlobalPageAllocTrait for GlobalPageAlloc { - fn global() -> Self { - GlobalPageAlloc - } +impl GlobalFrameAlloc for GlobalPageAlloc { + const GLOBAL: Self = GlobalPageAlloc; } -impl PageAlloc for BuddyPageAlloc { - type RawPage = RawPagePtr; +impl PageTableAlloc for GlobalPageAlloc { + type Folio = Folio; - fn alloc_order(&self, order: u32) -> Option<RawPagePtr> { - BUDDY_ALLOC.lock().alloc_order(order) + fn alloc(&self) -> Self::Folio { + FrameAlloc::alloc(self).unwrap() } - unsafe fn dealloc(&self, page_ptr: RawPagePtr) { - BUDDY_ALLOC.lock().dealloc(page_ptr); + unsafe fn from_raw(&self, pfn: PFN) -> Self::Folio { + unsafe { Folio::from_raw(pfn) } } +} - fn has_management_over(&self, page_ptr: RawPagePtr) -> bool { - BuddyAllocator::has_management_over(page_ptr) - } +#[define_late_init] +async fn buddyinfo_file() { + populate_root(Arc::from(&b"buddyinfo"[..]), |buffer| { + let stats = BUDDY_ALLOC.lock().dump_stat(); + + buf_write!(buffer, "Node 0, zone {:>10}", "Normal")?; + + for stat in &stats { + buf_write!(buffer, " {:>6}", stat.free_count)?; + } + + buf_write!(buffer, "\nNode 0, zone {:>10}", "Alloced")?; + + for stat in &stats { + buf_write!(buffer, " {:>6}", stat.alloced_count)?; + } + + buf_write!(buffer, "\n")?; + + Ok(()) + }) + .await; } diff --git a/src/kernel/mem/page_alloc/raw_page.rs b/src/kernel/mem/page_alloc/raw_page.rs index 54d4d590..16d57714 100644 --- a/src/kernel/mem/page_alloc/raw_page.rs +++ b/src/kernel/mem/page_alloc/raw_page.rs @@ -1,91 +1,58 @@ -use crate::kernel::mem::page_cache::PageCacheRawPage; -use crate::kernel::mem::PhysAccess; -use buddy_allocator::BuddyRawPage; -use core::{ - ptr::NonNull, - sync::atomic::{AtomicU32, AtomicUsize, Ordering}, -}; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; + +use buddy_allocator::BuddyFolio; use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::{ - address::{PAddr, PhysAccess as _}, - paging::{RawPage as RawPageTrait, PFN}, -}; -use intrusive_list::{container_of, Link}; -use slab_allocator::SlabRawPage; +use eonix_mm::address::{PAddr, PhysAccess as _}; +use eonix_mm::paging::{FolioList, FolioListSized, Zone, PFN}; +use intrusive_list::{container_of, Link, List}; +use slab_allocator::{SlabPage, SlabPageAlloc, SlabSlot}; -const PAGE_ARRAY: NonNull<RawPage> = - unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; +use super::zones::ZONE; +use super::{GlobalPageAlloc, PerCpuPage}; +use crate::kernel::mem::PhysAccess; pub struct PageFlags(AtomicU32); -struct SlabPageInner { - allocated_count: u32, - free_next: Option<NonNull<usize>>, +#[derive(Clone, Copy)] +struct SlabPageData { + allocated_count: usize, + free_next: Option<NonNull<SlabSlot>>, } -impl SlabPageInner { - fn new(free_next: Option<NonNull<usize>>) -> Self { +impl SlabPageData { + const fn new() -> Self { Self { allocated_count: 0, - free_next, + free_next: None, } } } -struct PageCacheInner { - valid_size: usize, -} - -pub struct BuddyPageInner {} - -enum PageType { - Buddy(BuddyPageInner), - Slab(SlabPageInner), - PageCache(PageCacheInner), -} - -impl PageType { - fn slab_data(&mut self) -> &mut SlabPageInner { - if let PageType::Slab(slab_data) = self { - return slab_data; - } else { - unreachable!() - } - } - - fn page_cache_data(&mut self) -> &mut PageCacheInner { - if let PageType::PageCache(cache_data) = self { - return cache_data; - } else { - unreachable!() - } - } +#[repr(C)] +union PageData { + slab: SlabPageData, } pub struct RawPage { /// This can be used for LRU page swap in the future. /// /// Now only used for free page links in the buddy system. - link: Link, + pub link: Link, /// # Safety /// This field is only used in buddy system and is protected by the global lock. - order: u32, - flags: PageFlags, - refcount: AtomicUsize, + pub order: u32, + pub flags: PageFlags, + pub refcount: AtomicUsize, - shared_data: PageType, + shared_data: PageData, } -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct RawPagePtr(NonNull<RawPage>); - impl PageFlags { - pub const PRESENT: u32 = 1 << 0; - // pub const LOCKED: u32 = 1 << 1; + pub const LOCKED: u32 = 1 << 1; pub const BUDDY: u32 = 1 << 2; pub const SLAB: u32 = 1 << 3; pub const DIRTY: u32 = 1 << 4; - pub const FREE: u32 = 1 << 5; pub const LOCAL: u32 = 1 << 6; pub fn has(&self, flag: u32) -> bool { @@ -99,174 +66,159 @@ impl PageFlags { pub fn clear(&self, flag: u32) { self.0.fetch_and(!flag, Ordering::Relaxed); } -} -impl RawPagePtr { - pub const fn new(ptr: NonNull<RawPage>) -> Self { - Self(ptr) - } - - /// Get a raw pointer to the underlying `RawPage` struct. + /// Set the flag and return whether it was already set. /// - /// # Safety - /// Doing arithmetic on the pointer returned will cause immediate undefined behavior. - pub const unsafe fn as_ptr(self) -> *mut RawPage { - self.0.as_ptr() - } - - pub const fn as_ref<'a>(self) -> &'a RawPage { - unsafe { &*self.as_ptr() } + /// If multiple flags are given, returns true if any of them were already set. + pub fn test_and_set(&self, flag: u32) -> bool { + (self.0.fetch_or(flag, Ordering::Relaxed) & flag) != 0 } +} - pub const fn as_mut<'a>(self) -> &'a mut RawPage { - unsafe { &mut *self.as_ptr() } +impl BuddyFolio for RawPage { + fn pfn(&self) -> PFN { + ZONE.get_pfn(self) } - pub const fn order(&self) -> u32 { - self.as_ref().order + fn get_order(&self) -> u32 { + self.order } - pub const fn flags(&self) -> &PageFlags { - &self.as_ref().flags + fn is_buddy(&self) -> bool { + self.flags.has(PageFlags::BUDDY) } - pub const fn refcount(&self) -> &AtomicUsize { - &self.as_ref().refcount + fn set_order(&mut self, order: u32) { + self.order = order; } - // return the ptr point to the actually raw page - pub fn real_ptr<T>(&self) -> NonNull<T> { - let pfn = unsafe { PFN::from(RawPagePtr(NonNull::new_unchecked(self.as_ptr()))) }; - unsafe { PAddr::from(pfn).as_ptr::<T>() } + fn set_buddy(&mut self, val: bool) { + if val { + self.flags.set(PageFlags::BUDDY); + } else { + self.flags.clear(PageFlags::BUDDY) + } } } -impl From<RawPagePtr> for PFN { - fn from(value: RawPagePtr) -> Self { - let idx = unsafe { value.as_ptr().offset_from(PAGE_ARRAY.as_ptr()) as usize }; - Self::from(idx) - } -} +impl SlabPage for RawPage { + fn get_data_ptr(&self) -> NonNull<[u8]> { + let paddr_start = PAddr::from(ZONE.get_pfn(self)); + let page_data_ptr = unsafe { paddr_start.as_ptr() }; -impl From<PFN> for RawPagePtr { - fn from(pfn: PFN) -> Self { - let raw_page_ptr = unsafe { PAGE_ARRAY.add(usize::from(pfn)) }; - Self::new(raw_page_ptr) + NonNull::slice_from_raw_parts(page_data_ptr, 1 << (self.order + 12)) } -} -impl RawPageTrait for RawPagePtr { - fn order(&self) -> u32 { - self.order() + fn get_free_slot(&self) -> Option<NonNull<SlabSlot>> { + unsafe { + // SAFETY: TODO + self.shared_data.slab.free_next + } } - fn refcount(&self) -> &AtomicUsize { - self.refcount() + fn set_free_slot(&mut self, next: Option<NonNull<SlabSlot>>) { + self.shared_data.slab.free_next = next; } - fn is_present(&self) -> bool { - self.flags().has(PageFlags::PRESENT) + fn get_alloc_count(&self) -> usize { + unsafe { + // SAFETY: TODO + self.shared_data.slab.allocated_count + } } -} -impl BuddyRawPage for RawPagePtr { - unsafe fn from_link(link: &mut Link) -> Self { - let raw_page_ptr = container_of!(link, RawPage, link); - Self(raw_page_ptr) - } + fn inc_alloc_count(&mut self) -> usize { + unsafe { + // SAFETY: TODO + self.shared_data.slab.allocated_count += 1; - fn set_order(&self, order: u32) { - self.as_mut().order = order; + self.shared_data.slab.allocated_count + } } - unsafe fn get_link(&self) -> &mut Link { - &mut self.as_mut().link - } + fn dec_alloc_count(&mut self) -> usize { + unsafe { + // SAFETY: TODO + self.shared_data.slab.allocated_count -= 1; - fn is_buddy(&self) -> bool { - self.flags().has(PageFlags::BUDDY) + self.shared_data.slab.allocated_count + } } - fn is_free(&self) -> bool { - self.flags().has(PageFlags::FREE) - } + unsafe fn from_allocated(ptr: NonNull<u8>) -> &'static mut Self { + unsafe { + // SAFETY: The caller ensures that `ptr` is valid. + let paddr = ArchPhysAccess::from_ptr(ptr); + let pfn = PFN::from(paddr); - fn set_buddy(&self) { - self.flags().set(PageFlags::BUDDY); + ZONE.get_page(pfn) + .expect("Page outside of the global zone") + .as_mut() + } } +} - fn set_free(&self) { - self.flags().set(PageFlags::FREE); +impl PerCpuPage for RawPage { + fn set_local(&mut self, val: bool) { + if val { + self.flags.set(PageFlags::LOCAL) + } else { + self.flags.clear(PageFlags::LOCAL) + } } +} - fn clear_buddy(&self) { - self.flags().clear(PageFlags::BUDDY); - } +pub struct RawPageList(List); - fn clear_free(&self) { - self.flags().clear(PageFlags::FREE); - } -} +unsafe impl Send for RawPageList {} -impl SlabRawPage for RawPagePtr { - unsafe fn from_link(link: &mut Link) -> Self { - let raw_page_ptr = container_of!(link, RawPage, link); - Self(raw_page_ptr) - } +impl FolioList for RawPageList { + type Folio = RawPage; - unsafe fn get_link(&self) -> &mut Link { - &mut self.as_mut().link + fn is_empty(&self) -> bool { + self.0.is_empty() } - fn in_which(ptr: *mut u8) -> RawPagePtr { + fn peek_head(&mut self) -> Option<&mut Self::Folio> { unsafe { - // SAFETY: The pointer is allocated from the slab allocator, - // which can't be null. - let ptr = NonNull::new_unchecked(ptr); + let link = self.0.head()?; + let mut raw_page_ptr = container_of!(link, RawPage, link); - // SAFETY: The pointer is valid. - let paddr = ArchPhysAccess::from_ptr(ptr); - let pfn = PFN::from(paddr); - - RawPagePtr::from(pfn) + Some(raw_page_ptr.as_mut()) } } - fn allocated_count(&self) -> &mut u32 { - &mut self.as_mut().shared_data.slab_data().allocated_count - } + fn pop_head(&mut self) -> Option<&'static mut Self::Folio> { + unsafe { + let link = self.0.pop()?; + let mut raw_page_ptr = container_of!(link, RawPage, link); - fn next_free(&self) -> &mut Option<NonNull<usize>> { - &mut self.as_mut().shared_data.slab_data().free_next + Some(raw_page_ptr.as_mut()) + } } - fn real_page_ptr(&self) -> *mut u8 { - self.real_ptr().as_ptr() + fn push_tail(&mut self, page: &'static mut Self::Folio) { + self.0.insert(&mut page.link); } - fn slab_init(&self, first_free: Option<NonNull<usize>>) { - self.as_mut().shared_data = PageType::Slab(SlabPageInner::new(first_free)); + fn remove(&mut self, page: &mut Self::Folio) { + self.0.remove(&mut page.link) } } -impl PageCacheRawPage for RawPagePtr { - fn valid_size(&self) -> &mut usize { - &mut self.as_mut().shared_data.page_cache_data().valid_size - } - - fn is_dirty(&self) -> bool { - self.flags().has(PageFlags::DIRTY) - } +impl FolioListSized for RawPageList { + const NEW: Self = RawPageList(List::new()); +} - fn clear_dirty(&self) { - self.flags().clear(PageFlags::DIRTY); - } +unsafe impl SlabPageAlloc for GlobalPageAlloc { + type Page = RawPage; + type PageList = RawPageList; - fn set_dirty(&self) { - self.flags().set(PageFlags::DIRTY); - } + fn alloc_slab_page(&self) -> &'static mut RawPage { + let raw_page = self.alloc_raw_order(0).expect("Out of memory"); + raw_page.flags.set(PageFlags::SLAB); + raw_page.shared_data.slab = SlabPageData::new(); - fn cache_init(&self) { - self.as_mut().shared_data = PageType::PageCache(PageCacheInner { valid_size: 0 }); + raw_page } } diff --git a/src/kernel/mem/page_alloc/zones.rs b/src/kernel/mem/page_alloc/zones.rs new file mode 100644 index 00000000..032b9cd0 --- /dev/null +++ b/src/kernel/mem/page_alloc/zones.rs @@ -0,0 +1,31 @@ +use core::ptr::NonNull; + +use eonix_mm::address::PRange; +use eonix_mm::paging::{Zone, PFN}; + +use super::RawPage; + +pub static ZONE: GlobalZone = GlobalZone(); + +const PAGE_ARRAY: NonNull<RawPage> = + unsafe { NonNull::new_unchecked(0xffffff8040000000 as *mut _) }; + +pub struct GlobalZone(); + +impl GlobalZone { + pub fn get_pfn(&self, page_ptr: *const RawPage) -> PFN { + PFN::from(unsafe { page_ptr.offset_from(PAGE_ARRAY.as_ptr()) as usize }) + } +} + +impl Zone for GlobalZone { + type Page = RawPage; + + fn contains_prange(&self, _: PRange) -> bool { + true + } + + fn get_page(&self, pfn: PFN) -> Option<NonNull<RawPage>> { + Some(unsafe { PAGE_ARRAY.add(usize::from(pfn)) }) + } +} diff --git a/src/kernel/mem/page_cache.rs b/src/kernel/mem/page_cache.rs index 3ccf3255..3fe33d5b 100644 --- a/src/kernel/mem/page_cache.rs +++ b/src/kernel/mem/page_cache.rs @@ -1,376 +1,235 @@ -use super::{paging::AllocZeroed, Page}; -use crate::{ - io::{Buffer, FillResult, Stream}, - kernel::mem::page_alloc::RawPagePtr, - prelude::KResult, - GlobalPageAlloc, -}; -use align_ext::AlignExt; -use alloc::{collections::btree_map::BTreeMap, sync::Weak}; -use core::mem::ManuallyDrop; -use eonix_hal::mm::ArchPhysAccess; -use eonix_mm::{ - address::{PAddr, PhysAccess}, - paging::{PageAlloc, RawPage, PAGE_SIZE, PAGE_SIZE_BITS, PFN}, -}; +use alloc::collections::btree_map::{BTreeMap, Entry}; +use core::future::Future; +use core::ops::{Deref, DerefMut}; + +use eonix_mm::paging::{Folio as _, PAGE_SIZE, PAGE_SIZE_BITS, PFN}; use eonix_sync::Mutex; -pub struct PageCache { - pages: Mutex<BTreeMap<usize, CachePage>>, - backend: Weak<dyn PageCacheBackend>, -} +use super::page_alloc::PageFlags; +use super::{Folio, FolioOwned}; +use crate::io::{Buffer, Stream}; +use crate::kernel::constants::EINVAL; +use crate::kernel::vfs::inode::InodeUse; +use crate::prelude::KResult; -unsafe impl Send for PageCache {} -unsafe impl Sync for PageCache {} +#[repr(transparent)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct PageOffset(usize); -#[derive(Clone, Copy)] -pub struct CachePage(RawPagePtr); +pub struct PageCache { + pages: Mutex<BTreeMap<PageOffset, CachePage>>, + inode: InodeUse, +} -unsafe impl Send for CachePage {} +pub struct CachePage(Folio); -impl Buffer for CachePage { - fn total(&self) -> usize { - PAGE_SIZE +impl PageOffset { + pub const fn from_byte_floor(offset: usize) -> Self { + Self(offset >> PAGE_SIZE_BITS) } - fn wrote(&self) -> usize { - self.valid_size() + pub const fn from_byte_ceil(offset: usize) -> Self { + Self((offset + PAGE_SIZE - 1) >> PAGE_SIZE_BITS) } - fn fill(&mut self, data: &[u8]) -> KResult<FillResult> { - let valid_size = self.valid_size(); - let available = &mut self.all_mut()[valid_size..]; - if available.len() == 0 { - return Ok(FillResult::Full); - } - - let len = core::cmp::min(data.len(), available.len()); - available[..len].copy_from_slice(&data[..len]); + pub fn iter_till(self, end: PageOffset) -> impl Iterator<Item = PageOffset> { + (self.0..end.0).map(PageOffset) + } - *self.0.valid_size() += len; + pub fn page_count(self) -> usize { + self.0 + } - if len < data.len() { - Ok(FillResult::Partial(len)) - } else { - Ok(FillResult::Done(len)) - } + pub fn byte_count(self) -> usize { + self.page_count() * PAGE_SIZE } } impl CachePage { pub fn new() -> Self { - let page = GlobalPageAlloc.alloc().unwrap(); - page.cache_init(); - Self(page) + CachePage(Folio::alloc()) } pub fn new_zeroed() -> Self { - let page = Page::zeroed(); - let raw_page_ptr = RawPagePtr::from(page.into_raw()); + CachePage({ + let mut folio = FolioOwned::alloc(); + folio.as_bytes_mut().fill(0); - raw_page_ptr.cache_init(); - Self(raw_page_ptr) + folio.share() + }) } - pub fn valid_size(&self) -> usize { - *self.0.valid_size() - } - - pub fn set_valid_size(&mut self, valid_size: usize) { - *self.0.valid_size() = valid_size; - } - - pub fn all(&self) -> &[u8] { - unsafe { - core::slice::from_raw_parts( - // SAFETY: The page is exclusively owned by us, so we can safely access its data. - ArchPhysAccess::as_ptr(PAddr::from(PFN::from(self.0))).as_ptr(), - PAGE_SIZE, - ) - } + pub fn is_dirty(&self) -> bool { + self.flags.has(PageFlags::DIRTY) } - pub fn all_mut(&mut self) -> &mut [u8] { - unsafe { - core::slice::from_raw_parts_mut( - // SAFETY: The page is exclusively owned by us, so we can safely access its data. - ArchPhysAccess::as_ptr(PAddr::from(PFN::from(self.0))).as_ptr(), - PAGE_SIZE, - ) + pub fn set_dirty(&self, dirty: bool) { + if dirty { + self.flags.set(PageFlags::DIRTY); + } else { + self.flags.clear(PageFlags::DIRTY); } } - pub fn valid_data(&self) -> &[u8] { - &self.all()[..self.valid_size()] + pub fn add_mapping(&self) -> PFN { + // TODO: Increase map_count + self.0.clone().into_raw() } +} - pub fn is_dirty(&self) -> bool { - self.0.is_dirty() - } +impl Deref for CachePage { + type Target = Folio; - pub fn set_dirty(&self) { - self.0.set_dirty(); + fn deref(&self) -> &Self::Target { + &self.0 } +} - pub fn clear_dirty(&self) { - self.0.clear_dirty(); +impl DerefMut for CachePage { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 } } impl PageCache { - pub fn new(backend: Weak<dyn PageCacheBackend>) -> Self { + pub fn new(inode: InodeUse) -> Self { Self { pages: Mutex::new(BTreeMap::new()), - backend: backend, + inode, } } - pub async fn read(&self, buffer: &mut dyn Buffer, mut offset: usize) -> KResult<usize> { - let mut pages = self.pages.lock().await; - let size = self.backend.upgrade().unwrap().size(); - - loop { - if offset >= size { - break; - } - let page_id = offset >> PAGE_SIZE_BITS; - let page = pages.get(&page_id); - - match page { - Some(page) => { - let inner_offset = offset % PAGE_SIZE; - let available_in_file = size.saturating_sub(offset); - - // TODO: still cause unnecessary IO if valid_size < PAGESIZE - // and fill result is Done - let page_data = &page.valid_data()[inner_offset..]; - let read_size = page_data.len().min(available_in_file); - - if read_size == 0 - || buffer.fill(&page_data[..read_size])?.should_stop() - || buffer.available() == 0 - { - break; - } - offset += read_size; - } - None => { + pub fn get_page_locked<'a>( + &self, + pages: &'a mut BTreeMap<PageOffset, CachePage>, + pgoff: PageOffset, + ) -> impl Future<Output = KResult<&'a mut CachePage>> + Send + use<'_, 'a> { + async move { + match pages.entry(pgoff) { + Entry::Occupied(ent) => Ok(ent.into_mut()), + Entry::Vacant(vacant_entry) => { let mut new_page = CachePage::new(); - self.backend - .upgrade() - .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE))?; - pages.insert(page_id, new_page); + self.inode.read_page(&mut new_page, pgoff).await?; + + Ok(vacant_entry.insert(new_page)) } } } + } - Ok(buffer.wrote()) + fn len(&self) -> usize { + self.inode.info.lock().size as usize } - pub async fn write(&self, stream: &mut dyn Stream, mut offset: usize) -> KResult<usize> { + // TODO: Remove this. + pub async fn with_page(&self, pgoff: PageOffset, func: impl FnOnce(&CachePage)) -> KResult<()> { let mut pages = self.pages.lock().await; - let old_size = self.backend.upgrade().unwrap().size(); - let mut wrote = 0; - - loop { - let page_id = offset >> PAGE_SIZE_BITS; - let page = pages.get_mut(&page_id); - - match page { - Some(page) => { - let inner_offset = offset % PAGE_SIZE; - let cursor_end = match stream.poll_data(&mut page.all_mut()[inner_offset..])? { - Some(buf) => { - wrote += buf.len(); - inner_offset + buf.len() - } - None => { - break; - } - }; - - if page.valid_size() < cursor_end { - page.set_valid_size(cursor_end); - } - page.set_dirty(); - offset += PAGE_SIZE - inner_offset; - } - None => { - let new_page = if (offset >> PAGE_SIZE_BITS) > (old_size >> PAGE_SIZE_BITS) { - let new_page = CachePage::new_zeroed(); - new_page - } else { - let mut new_page = CachePage::new(); - self.backend - .upgrade() - .unwrap() - .read_page(&mut new_page, offset.align_down(PAGE_SIZE))?; - new_page - }; - - pages.insert(page_id, new_page); - } - } + if pgoff > PageOffset::from_byte_ceil(self.len()) { + return Err(EINVAL); } - Ok(wrote) - } + let cache_page = self.get_page_locked(&mut pages, pgoff).await?; + + func(cache_page); - pub async fn fsync(&self) -> KResult<()> { - let pages = self.pages.lock().await; - for (page_id, page) in pages.iter() { - if page.is_dirty() { - self.backend - .upgrade() - .unwrap() - .write_page(&mut CachePageStream::new(*page), page_id << PAGE_SIZE_BITS)?; - page.clear_dirty(); - } - } Ok(()) } - // This function is used for extend write or truncate - pub async fn resize(&self, new_size: usize) -> KResult<()> { + pub async fn read(&self, buffer: &mut dyn Buffer, mut offset: usize) -> KResult<usize> { let mut pages = self.pages.lock().await; - let old_size = self.backend.upgrade().unwrap().size(); - - if new_size < old_size { - let begin = new_size.align_down(PAGE_SIZE) >> PAGE_SIZE_BITS; - let end = old_size.align_up(PAGE_SIZE) >> PAGE_SIZE_BITS; - - for page_id in begin..end { - pages.remove(&page_id); - } - } else if new_size > old_size { - let begin = old_size.align_down(PAGE_SIZE) >> PAGE_SIZE_BITS; - let end = new_size.align_up(PAGE_SIZE) >> PAGE_SIZE_BITS; - - pages.remove(&begin); - - for page_id in begin..end { - let mut new_page = CachePage::new_zeroed(); + let total_len = self.len(); - if page_id != end - 1 { - new_page.set_valid_size(PAGE_SIZE); - } else { - new_page.set_valid_size(new_size % PAGE_SIZE); - } - new_page.set_dirty(); - pages.insert(page_id, new_page); - } + if offset >= total_len { + return Ok(0); } - Ok(()) - } + let pgoff_start = PageOffset::from_byte_floor(offset); + let pgoff_end = PageOffset::from_byte_ceil(total_len); - pub async fn with_page<F, O>(&self, offset: usize, func: F) -> KResult<Option<O>> - where - F: FnOnce(&Page, &CachePage) -> O, - { - let offset_aligin = offset.align_down(PAGE_SIZE); - let page_id = offset_aligin >> PAGE_SIZE_BITS; - let size = self.backend.upgrade().unwrap().size(); + for pgoff in pgoff_start.iter_till(pgoff_end) { + let page = self.get_page_locked(&mut pages, pgoff).await?; - if offset_aligin > size { - return Ok(None); - } + let end_offset = (offset + PAGE_SIZE) / PAGE_SIZE * PAGE_SIZE; + let real_end = end_offset.min(total_len); - let mut pages = self.pages.lock().await; + let inner_offset = offset % PAGE_SIZE; + let data_len = real_end - offset; - let raw_page_ptr = match pages.get(&page_id) { - Some(CachePage(raw_page_ptr)) => *raw_page_ptr, - None => { - let mut new_page = CachePage::new(); - self.backend - .upgrade() - .unwrap() - .read_page(&mut new_page, offset_aligin)?; - pages.insert(page_id, new_page); - new_page.0 + if buffer + .fill(&page.lock().as_bytes()[inner_offset..inner_offset + data_len])? + .should_stop() + || buffer.available() == 0 + { + break; } - }; - - unsafe { - let page = ManuallyDrop::new(Page::from_raw_unchecked(PFN::from(raw_page_ptr))); - Ok(Some(func(&page, &CachePage(raw_page_ptr)))) + offset = real_end; } - } -} -pub struct CachePageStream { - page: CachePage, - cur: usize, -} - -impl CachePageStream { - pub fn new(page: CachePage) -> Self { - Self { page, cur: 0 } + Ok(buffer.wrote()) } - pub fn remaining(&self) -> usize { - self.page.valid_size().saturating_sub(self.cur) - } + pub async fn write(&self, stream: &mut dyn Stream, mut offset: usize) -> KResult<usize> { + let mut pages = self.pages.lock().await; + let mut total_written = 0; - pub fn is_drained(&self) -> bool { - self.cur >= self.page.valid_size() - } -} + loop { + let end_offset = (offset + PAGE_SIZE) / PAGE_SIZE * PAGE_SIZE; + let len = end_offset - offset; + + // TODO: Rewrite to return a write state object. + let page = self + .inode + .write_begin(self, &mut pages, offset, len) + .await?; + + let inner_offset = offset % PAGE_SIZE; + let written = stream + .poll_data(&mut page.lock().as_bytes_mut()[inner_offset..])? + .map(|b| b.len()) + .unwrap_or(0); + + page.set_dirty(true); + self.inode + .write_end(self, &mut pages, offset, len, written) + .await?; + + if written == 0 { + break; + } -impl Stream for CachePageStream { - fn poll_data<'a>(&mut self, buf: &'a mut [u8]) -> KResult<Option<&'a mut [u8]>> { - if self.cur >= self.page.valid_size() { - return Ok(None); + total_written += written; + offset += written; } - let page_data = &self.page.all()[self.cur..self.page.valid_size()]; - let to_read = buf.len().min(page_data.len()); + Ok(total_written) + } - buf[..to_read].copy_from_slice(&page_data[..to_read]); - self.cur += to_read; + pub async fn fsync(&self) -> KResult<()> { + let mut pages = self.pages.lock().await; - Ok(Some(&mut buf[..to_read])) - } + for (&pgoff, page) in pages.iter_mut() { + if !page.is_dirty() { + continue; + } - fn ignore(&mut self, len: usize) -> KResult<Option<usize>> { - if self.cur >= self.page.valid_size() { - return Ok(None); + self.inode.write_page(page, pgoff).await?; + page.set_dirty(false); } - let to_ignore = len.min(self.page.valid_size() - self.cur); - self.cur += to_ignore; - Ok(Some(to_ignore)) + Ok(()) } } -// with this trait, "page cache" and "block cache" are unified, -// for fs, offset is file offset (floor algin to PAGE_SIZE) -// for blkdev, offset is block idx (floor align to PAGE_SIZE / BLK_SIZE) -// Oh no, this would make unnecessary cache -pub trait PageCacheBackend { - fn read_page(&self, page: &mut CachePage, offset: usize) -> KResult<usize>; - - fn write_page(&self, page: &mut CachePageStream, offset: usize) -> KResult<usize>; - - fn size(&self) -> usize; -} - -pub trait PageCacheRawPage: RawPage { - fn valid_size(&self) -> &mut usize; - - fn is_dirty(&self) -> bool; - - fn set_dirty(&self); - - fn clear_dirty(&self); - - fn cache_init(&self); +impl core::fmt::Debug for PageCache { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("PageCache").finish() + } } impl Drop for PageCache { fn drop(&mut self) { + // XXX: Send the PageCache to some flusher worker. let _ = self.fsync(); } } diff --git a/src/kernel/mem/paging.rs b/src/kernel/mem/paging.rs index 8c5f41f2..1b95ce79 100644 --- a/src/kernel/mem/paging.rs +++ b/src/kernel/mem/paging.rs @@ -1,12 +1,11 @@ -use super::{access::AsMemoryBlock, page_alloc::GlobalPageAlloc, MemoryBlock, PhysAccess}; -use crate::io::{Buffer, FillResult}; -use eonix_mm::paging::{Page as GenericPage, PageAlloc}; +use eonix_mm::paging::Folio as _; -pub type Page = GenericPage<GlobalPageAlloc>; +use super::folio::FolioOwned; +use crate::io::{Buffer, FillResult}; /// A buffer that wraps a page and provides a `Buffer` interface. pub struct PageBuffer { - page: Page, + page: FolioOwned, offset: usize, } @@ -14,28 +13,16 @@ pub trait AllocZeroed { fn zeroed() -> Self; } -impl<A: PageAlloc> AsMemoryBlock for GenericPage<A> { - fn as_memblk(&self) -> MemoryBlock { - unsafe { - // SAFETY: `self.start()` points to valid memory of length `self.len()`. - MemoryBlock::new(self.start().as_ptr::<()>().addr(), self.len()) - } - } -} - impl PageBuffer { pub fn new() -> Self { Self { - page: Page::alloc(), + page: FolioOwned::alloc(), offset: 0, } } pub fn all(&self) -> &[u8] { - unsafe { - // SAFETY: The page is exclusivly owned by us. - self.page.as_memblk().as_bytes() - } + self.page.as_bytes() } pub fn data(&self) -> &[u8] { @@ -43,10 +30,7 @@ impl PageBuffer { } pub fn available_mut(&mut self) -> &mut [u8] { - unsafe { - // SAFETY: The page is exclusivly owned by us. - &mut self.page.as_memblk().as_bytes_mut()[self.offset..] - } + &mut self.page.as_bytes_mut()[self.offset..] } } @@ -76,14 +60,3 @@ impl Buffer for PageBuffer { } } } - -impl AllocZeroed for Page { - fn zeroed() -> Self { - let page = Self::alloc(); - unsafe { - // SAFETY: The page is exclusivly owned by us. - page.as_memblk().as_bytes_mut().fill(0); - } - page - } -} diff --git a/src/kernel/pcie/device.rs b/src/kernel/pcie/device.rs index 085e7b9a..2a8f150d 100644 --- a/src/kernel/pcie/device.rs +++ b/src/kernel/pcie/device.rs @@ -1,14 +1,17 @@ -use super::{ - header::{Bar, Command}, - CommonHeader, Header, -}; -use crate::kernel::mem::PhysAccess as _; +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::num::NonZero; +use core::ops::RangeInclusive; + use align_ext::AlignExt; -use alloc::{collections::btree_map::BTreeMap, sync::Arc, vec::Vec}; -use core::{num::NonZero, ops::RangeInclusive}; use eonix_mm::address::{Addr, PAddr, PRange}; use eonix_sync::Spin; +use super::header::{Bar, Command}; +use super::{CommonHeader, Header}; +use crate::kernel::mem::PhysAccess as _; + pub(super) static PCIE_DEVICES: Spin<BTreeMap<u32, Vec<Arc<PCIDevice>>>> = Spin::new(BTreeMap::new()); @@ -20,7 +23,7 @@ pub struct PCIDevice<'a> { pub device_id: u16, } -#[allow(dead_code)] +#[allow(unused)] #[derive(Clone)] pub struct SegmentGroup { id: usize, @@ -28,6 +31,7 @@ pub struct SegmentGroup { base_address: PAddr, } +#[allow(unused)] #[derive(Clone)] pub struct ConfigSpace { pub bus: u8, @@ -180,10 +184,12 @@ impl PCIDevice<'_> { ); } + #[allow(unused)] pub fn config_space(&self) -> &ConfigSpace { &self.config_space } + #[allow(unused)] pub fn segment_group(&self) -> &SegmentGroup { &self.segment_group } @@ -209,7 +215,7 @@ impl PciMemoryAllocator { self.start += size; eonix_log::println_trace!( - "trace_pci", + feat: "trace_pci", "PciMemoryAllocator: Allocated {} bytes at {:#x}", size, base diff --git a/src/kernel/pcie/driver.rs b/src/kernel/pcie/driver.rs index be88b7df..eebaa896 100644 --- a/src/kernel/pcie/driver.rs +++ b/src/kernel/pcie/driver.rs @@ -4,21 +4,24 @@ use super::{ }; use crate::{kernel::constants::EEXIST, KResult}; use alloc::{ + boxed::Box, collections::btree_map::{self, BTreeMap}, sync::Arc, }; +use async_trait::async_trait; use eonix_sync::Spin; static PCIE_DRIVERS: Spin<BTreeMap<u32, Arc<dyn PCIDriver>>> = Spin::new(BTreeMap::new()); +#[async_trait] pub trait PCIDriver: Send + Sync { fn vendor_id(&self) -> u16; fn device_id(&self) -> u16; - fn handle_device(&self, device: Arc<PCIDevice<'static>>) -> Result<(), PciError>; + async fn handle_device(&self, device: Arc<PCIDevice<'static>>) -> Result<(), PciError>; } -pub fn register_driver(driver: impl PCIDriver + 'static) -> KResult<()> { +pub async fn register_driver(driver: impl PCIDriver + 'static) -> KResult<()> { let index = (driver.vendor_id() as u32) << 16 | driver.device_id() as u32; let driver = Arc::new(driver); @@ -31,7 +34,7 @@ pub fn register_driver(driver: impl PCIDriver + 'static) -> KResult<()> { let devices = PCIE_DEVICES.lock().get(&index).cloned(); if let Some(devices) = devices { for device in devices { - driver.handle_device(device)?; + driver.handle_device(device).await?; } }; diff --git a/src/kernel/pcie/header.rs b/src/kernel/pcie/header.rs index 889795d3..0a44ea28 100644 --- a/src/kernel/pcie/header.rs +++ b/src/kernel/pcie/header.rs @@ -1,10 +1,9 @@ +use core::marker::PhantomData; +use core::num::NonZero; +use core::ops::{BitAnd, BitOr, Deref, Not}; +use core::sync::atomic::{AtomicU16, AtomicU32, Ordering}; + use bitflags::bitflags; -use core::{ - marker::PhantomData, - num::NonZero, - ops::{BitAnd, BitOr, Deref, Not}, - sync::atomic::{AtomicU16, AtomicU32, Ordering}, -}; use eonix_hal::fence::memory_barrier; pub trait BitFlag: Sized + Copy { @@ -215,14 +214,14 @@ where } impl CommonHeader { - pub fn command(&self) -> Register<Command> { + pub fn command(&self) -> Register<'_, Command> { Register { register: unsafe { AtomicU16::from_ptr((&raw const self._command) as *mut u16) }, _phantom: PhantomData, } } - pub fn status(&self) -> Register<Status> { + pub fn status(&self) -> Register<'_, Status> { Register { register: unsafe { AtomicU16::from_ptr((&raw const self._status) as *mut u16) }, _phantom: PhantomData, @@ -231,7 +230,7 @@ impl CommonHeader { } impl Bars<'_> { - pub fn iter(&self) -> impl Iterator<Item = BarsEntry> + '_ { + pub fn iter(&self) -> impl Iterator<Item = BarsEntry<'_>> + use<'_> { struct BarsIterator<'a> { bars: &'a [AtomicU32], pos: usize, diff --git a/src/kernel/pcie/init.rs b/src/kernel/pcie/init.rs index c0253f4e..74a490b4 100644 --- a/src/kernel/pcie/init.rs +++ b/src/kernel/pcie/init.rs @@ -1,14 +1,16 @@ -use super::{ - device::{PCIDevice, SegmentGroup, PCIE_DEVICES}, - error::PciError, -}; -use crate::kernel::{mem::PhysAccess as _, pcie::device::PciMemoryAllocator}; -use acpi::{AcpiHandler, PhysicalMapping}; use alloc::collections::btree_map::Entry; use alloc::vec; + +use acpi::{AcpiHandler, PhysicalMapping}; use eonix_log::println_trace; -use eonix_mm::address::PAddr; +use eonix_mm::address::{PAddr, PRange}; + +use super::device::{PCIDevice, SegmentGroup, PCIE_DEVICES}; +use super::error::PciError; +use crate::kernel::mem::PhysAccess as _; +use crate::kernel::pcie::device::PciMemoryAllocator; +#[allow(unused)] #[derive(Clone)] struct AcpiHandlerImpl; @@ -33,7 +35,6 @@ pub fn init_pcie() -> Result<(), PciError> { #[cfg(target_arch = "x86_64")] { use acpi::{AcpiTables, PciConfigRegions}; - use eonix_mm::address::PAddr; let acpi_tables = unsafe { // SAFETY: Our impl should be correct. @@ -67,9 +68,9 @@ pub fn init_pcie() -> Result<(), PciError> { #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { - use crate::kernel::constants::{EINVAL, EIO, ENOENT}; use eonix_hal::arch_exported::fdt::FDT; - use eonix_mm::address::PRange; + + use crate::kernel::constants::{EINVAL, EIO, ENOENT}; let pcie_node = FDT .find_compatible(&["pci-host-ecam-generic"]) @@ -86,7 +87,7 @@ pub fn init_pcie() -> Result<(), PciError> { let size = u64::from_be_bytes(entry[20..28].try_into().unwrap()); println_trace!( - "trace_pci", + feat: "trace_pci", "PCIe range: PCI address = {:#x}, CPU address = {:#x}, size = {:#x}", pci_address, cpu_address, diff --git a/src/kernel/power.rs b/src/kernel/power.rs new file mode 100644 index 00000000..7cbe3a08 --- /dev/null +++ b/src/kernel/power.rs @@ -0,0 +1,46 @@ +use core::sync::atomic::{AtomicUsize, Ordering}; + +use eonix_hal::processor::{halt, CPU, CPU_COUNT}; +use eonix_log::println_info; + +static CPU_SHUTTING_DOWN: AtomicUsize = AtomicUsize::new(0); + +fn shutdown() { + #[cfg(arch_has_shutdown)] + { + eonix_hal::arch_exported::bootstrap::shutdown(); + } + #[cfg(not(arch_has_shutdown))] + { + println_info!("Shutdown not supported on this architecture..."); + } +} + +pub fn shutdown_system_nowait() { + shutdown(); + + loop { + halt(); + } +} + +pub fn shutdown_system() -> ! { + // TODO: We don't have IPI system for now. + shutdown_system_nowait(); + + let cpu_count = CPU_COUNT.load(Ordering::Relaxed); + + if CPU_SHUTTING_DOWN.fetch_add(1, Ordering::AcqRel) + 1 == cpu_count { + println_info!("All CPUs are shutting down. Gracefully powering off..."); + shutdown(); + } else { + println_info!( + "CPU {} is shutting down. Waiting for other CPUs...", + CPU::local().cpuid() + ); + } + + loop { + halt(); + } +} diff --git a/src/kernel/syscall.rs b/src/kernel/syscall.rs index 78ddcd1c..d06c5d88 100644 --- a/src/kernel/syscall.rs +++ b/src/kernel/syscall.rs @@ -1,11 +1,17 @@ -use super::task::ThreadAlloc; -use crate::kernel::task::Thread; use alloc::boxed::Box; -use core::{future::Future, marker::PhantomData, ops::Deref, pin::Pin}; +use core::future::Future; +use core::marker::PhantomData; +use core::ops::Deref; +use core::pin::Pin; + +use eonix_hal::extern_symbol_addr; use eonix_mm::address::{Addr, VAddr}; use eonix_sync::LazyLock; use posix_types::ctypes::PtrT; +use super::task::ThreadAlloc; +use crate::kernel::task::Thread; + pub mod file_rw; pub mod mm; pub mod net; @@ -280,12 +286,6 @@ impl<T> core::fmt::Debug for UserMut<T> { } static SYSCALL_HANDLERS: LazyLock<[Option<SyscallHandler>; MAX_SYSCALL_NO]> = LazyLock::new(|| { - extern "C" { - // SAFETY: `SYSCALL_HANDLERS` is defined in linker script. - fn RAW_SYSCALL_HANDLERS(); - fn RAW_SYSCALL_HANDLERS_SIZE(); - } - // DO NOT TOUCH THESE FUNCTIONS!!! // THEY ARE USED FOR KEEPING THE OBJECTS NOT STRIPPED BY THE LINKER!!! file_rw::keep_alive(); @@ -294,15 +294,14 @@ static SYSCALL_HANDLERS: LazyLock<[Option<SyscallHandler>; MAX_SYSCALL_NO]> = La procops::keep_alive(); sysinfo::keep_alive(); - let raw_handlers_addr = RAW_SYSCALL_HANDLERS as *const (); - let raw_handlers_size_byte = RAW_SYSCALL_HANDLERS_SIZE as usize; + let raw_handlers_size_byte = extern_symbol_addr!(RAW_SYSCALL_HANDLERS_SIZE); assert!(raw_handlers_size_byte % size_of::<RawSyscallHandler>() == 0); let raw_handlers_count = raw_handlers_size_byte / size_of::<RawSyscallHandler>(); let raw_handlers = unsafe { core::slice::from_raw_parts( - raw_handlers_addr as *const RawSyscallHandler, + extern_symbol_addr!(RAW_SYSCALL_HANDLERS, RawSyscallHandler), raw_handlers_count, ) }; diff --git a/src/kernel/syscall/file_rw.rs b/src/kernel/syscall/file_rw.rs index 1a48b255..8ac9c22a 100644 --- a/src/kernel/syscall/file_rw.rs +++ b/src/kernel/syscall/file_rw.rs @@ -1,34 +1,33 @@ -use super::{FromSyscallArg, User}; -use crate::io::IntoStream; -use crate::kernel::constants::{ - EBADF, EFAULT, EINVAL, ENOENT, ENOSYS, ENOTDIR, SEEK_CUR, SEEK_END, SEEK_SET, -}; -use crate::kernel::syscall::UserMut; -use crate::kernel::task::Thread; -use crate::kernel::timer::sleep; -use crate::kernel::vfs::filearray::FD; -use crate::kernel::vfs::inode::Mode; -use crate::kernel::vfs::{PollEvent, SeekOption}; -use crate::{ - io::{Buffer, BufferFill}, - kernel::{ - user::{CheckedUserPointer, UserBuffer, UserPointer, UserPointerMut, UserString}, - vfs::dentry::Dentry, - }, - path::Path, - prelude::*, -}; use alloc::sync::Arc; use core::time::Duration; + use posix_types::ctypes::{Long, PtrT}; use posix_types::namei::RenameFlags; use posix_types::open::{AtFlags, OpenFlags}; use posix_types::poll::FDSet; use posix_types::signal::{SigSet, Signal}; -use posix_types::stat::Stat; -use posix_types::stat::{StatX, TimeSpec}; +use posix_types::stat::{Stat, StatX, TimeSpec}; use posix_types::syscall_no::*; +use super::{FromSyscallArg, User}; +use crate::io::{Buffer, BufferFill, IntoStream}; +use crate::kernel::constants::{ + EBADF, EFAULT, EINVAL, ENOENT, ENOSYS, ENOTDIR, SEEK_CUR, SEEK_END, + SEEK_SET, +}; +use crate::kernel::syscall::UserMut; +use crate::kernel::task::Thread; +use crate::kernel::timer::sleep; +use crate::kernel::user::{ + CheckedUserPointer, UserBuffer, UserPointer, UserPointerMut, UserString, +}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::filearray::FD; +use crate::kernel::vfs::types::{DeviceId, Mode}; +use crate::kernel::vfs::{PollEvent, SeekOption}; +use crate::path::Path; +use crate::prelude::*; + impl FromSyscallArg for OpenFlags { fn from_arg(value: usize) -> Self { OpenFlags::from_bits_retain(value as u32) @@ -41,7 +40,7 @@ impl FromSyscallArg for AtFlags { } } -fn dentry_from( +async fn dentry_from( thread: &Thread, dirfd: FD, pathname: User<u8>, @@ -52,7 +51,7 @@ fn dentry_from( match (path.as_cstr().to_bytes_with_nul()[0], dirfd) { (b'/', _) | (_, FD::AT_FDCWD) => { let path = Path::new(path.as_cstr().to_bytes())?; - Dentry::open(&thread.fs_context, path, follow_symlink) + Dentry::open(&thread.fs_context, path, follow_symlink).await } (0, dirfd) => { let dir_file = thread.files.get(dirfd).ok_or(EBADF)?; @@ -63,7 +62,13 @@ fn dentry_from( let dir_file = thread.files.get(dirfd).ok_or(EBADF)?; let dir_dentry = dir_file.as_path().ok_or(ENOTDIR)?; - Dentry::open_at(&thread.fs_context, dir_dentry, path, follow_symlink) + Dentry::open_at( + &thread.fs_context, + dir_dentry, + path, + follow_symlink, + ) + .await } } } @@ -81,7 +86,12 @@ async fn read(fd: FD, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> { } #[eonix_macros::define_syscall(SYS_PREAD64)] -async fn pread64(fd: FD, buffer: UserMut<u8>, bufsize: usize, offset: usize) -> KResult<usize> { +async fn pread64( + fd: FD, + buffer: UserMut<u8>, + bufsize: usize, + offset: usize, +) -> KResult<usize> { let mut buffer = UserBuffer::new(buffer, bufsize)?; thread @@ -106,7 +116,12 @@ async fn write(fd: FD, buffer: User<u8>, count: usize) -> KResult<usize> { } #[eonix_macros::define_syscall(SYS_PWRITE64)] -async fn pwrite64(fd: FD, buffer: User<u8>, count: usize, offset: usize) -> KResult<usize> { +async fn pwrite64( + fd: FD, + buffer: User<u8>, + count: usize, + offset: usize, +) -> KResult<usize> { let buffer = CheckedUserPointer::new(buffer, count)?; let mut stream = buffer.into_stream(); @@ -119,18 +134,22 @@ async fn pwrite64(fd: FD, buffer: User<u8>, count: usize, offset: usize) -> KRes } #[eonix_macros::define_syscall(SYS_OPENAT)] -async fn openat(dirfd: FD, pathname: User<u8>, flags: OpenFlags, mut mode: Mode) -> KResult<FD> { - let dentry = dentry_from(thread, dirfd, pathname, flags.follow_symlink())?; - - let umask = *thread.fs_context.umask.lock(); - mode.mask_perm(!umask.non_format_bits()); +async fn openat( + dirfd: FD, + pathname: User<u8>, + flags: OpenFlags, + mode: Mode, +) -> KResult<FD> { + let dentry = + dentry_from(thread, dirfd, pathname, flags.follow_symlink()).await?; + let perm = mode.perm().mask_with(*thread.fs_context.umask.lock()); - thread.files.open(&dentry, flags, mode) + thread.files.open(thread, &dentry, flags, perm).await } #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_OPEN)] -async fn open(path: User<u8>, flags: OpenFlags, mode: u32) -> KResult<FD> { +async fn open(path: User<u8>, flags: OpenFlags, mode: Mode) -> KResult<FD> { sys_openat(thread, FD::AT_FDCWD, path, flags, mode).await } @@ -147,7 +166,10 @@ async fn dup(fd: FD) -> KResult<FD> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_DUP2)] async fn dup2(old_fd: FD, new_fd: FD) -> KResult<FD> { - thread.files.dup_to(old_fd, new_fd, OpenFlags::empty()) + thread + .files + .dup_to(old_fd, new_fd, OpenFlags::empty()) + .await } #[eonix_macros::define_syscall(SYS_DUP3)] @@ -157,7 +179,8 @@ async fn dup3(old_fd: FD, new_fd: FD, flags: OpenFlags) -> KResult<FD> { #[eonix_macros::define_syscall(SYS_PIPE2)] async fn pipe2(pipe_fd: UserMut<[FD; 2]>, flags: OpenFlags) -> KResult<()> { - let mut buffer = UserBuffer::new(pipe_fd.cast(), core::mem::size_of::<[FD; 2]>())?; + let mut buffer = + UserBuffer::new(pipe_fd.cast(), core::mem::size_of::<[FD; 2]>())?; let (read_fd, write_fd) = thread.files.pipe(flags)?; buffer.copy(&[read_fd, write_fd])?.ok_or(EFAULT) @@ -171,15 +194,29 @@ async fn pipe(pipe_fd: UserMut<[FD; 2]>) -> KResult<()> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_GETDENTS)] -async fn getdents(fd: FD, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> { +async fn getdents( + fd: FD, + buffer: UserMut<u8>, + bufsize: usize, +) -> KResult<usize> { let mut buffer = UserBuffer::new(buffer, bufsize)?; - thread.files.get(fd).ok_or(EBADF)?.getdents(&mut buffer)?; + thread + .files + .get(fd) + .ok_or(EBADF)? + .getdents(&mut buffer) + .await?; + Ok(buffer.wrote()) } #[eonix_macros::define_syscall(SYS_GETDENTS64)] -async fn getdents64(fd: FD, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> { +async fn getdents64( + fd: FD, + buffer: UserMut<u8>, + bufsize: usize, +) -> KResult<usize> { let mut buffer = UserBuffer::new(buffer, bufsize)?; thread @@ -206,7 +243,7 @@ async fn newfstatat( let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; let statbuf = UserPointerMut::new(statbuf)?; @@ -225,7 +262,8 @@ async fn newfstatat( )] #[cfg_attr(target_arch = "x86_64", eonix_macros::define_syscall(SYS_FSTAT64))] async fn newfstat(fd: FD, statbuf: UserMut<Stat>) -> KResult<()> { - sys_newfstatat(thread, fd, User::null(), statbuf, AtFlags::AT_EMPTY_PATH).await + sys_newfstatat(thread, fd, User::null(), statbuf, AtFlags::AT_EMPTY_PATH) + .await } #[eonix_macros::define_syscall(SYS_STATX)] @@ -247,7 +285,7 @@ async fn statx( let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; dentry.statx(&mut statx, mask)?; @@ -257,24 +295,23 @@ async fn statx( } #[eonix_macros::define_syscall(SYS_MKDIRAT)] -async fn mkdirat(dirfd: FD, pathname: User<u8>, mut mode: Mode) -> KResult<()> { - let umask = *thread.fs_context.umask.lock(); - mode.mask_perm(!umask.non_format_bits()); +async fn mkdirat(dirfd: FD, pathname: User<u8>, mode: Mode) -> KResult<()> { + let dentry = dentry_from(thread, dirfd, pathname, true).await?; + let perm = mode.perm().mask_with(*thread.fs_context.umask.lock()); - let dentry = dentry_from(thread, dirfd, pathname, true)?; - dentry.mkdir(mode) + dentry.mkdir(perm).await } #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_MKDIR)] -async fn mkdir(pathname: User<u8>, mode: u32) -> KResult<()> { +async fn mkdir(pathname: User<u8>, mode: Mode) -> KResult<()> { sys_mkdirat(thread, FD::AT_FDCWD, pathname, mode).await } #[eonix_macros::define_syscall(SYS_FTRUNCATE64)] async fn truncate64(fd: FD, length: usize) -> KResult<()> { let file = thread.files.get(fd).ok_or(EBADF)?; - file.as_path().ok_or(EBADF)?.truncate(length) + file.as_path().ok_or(EBADF)?.truncate(length).await } #[cfg(target_arch = "x86_64")] @@ -283,53 +320,87 @@ async fn truncate(pathname: User<u8>, length: usize) -> KResult<()> { let path = UserString::new(pathname)?; let path = Path::new(path.as_cstr().to_bytes())?; - let dentry = Dentry::open(&thread.fs_context, path, true)?; + let dentry = Dentry::open(&thread.fs_context, path, true).await?; - dentry.truncate(length) + dentry.truncate(length).await } #[eonix_macros::define_syscall(SYS_UNLINKAT)] async fn unlinkat(dirfd: FD, pathname: User<u8>) -> KResult<()> { - dentry_from(thread, dirfd, pathname, false)?.unlink() + dentry_from(thread, dirfd, pathname, false) + .await? + .unlink() + .await } #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_UNLINK)] async fn unlink(pathname: User<u8>) -> KResult<()> { - sys_unlinkat(thread, FD::AT_FDCWD, pathname) + sys_unlinkat(thread, FD::AT_FDCWD, pathname).await } #[eonix_macros::define_syscall(SYS_SYMLINKAT)] -async fn symlinkat(target: User<u8>, dirfd: FD, linkpath: User<u8>) -> KResult<()> { +async fn symlinkat( + target: User<u8>, + dirfd: FD, + linkpath: User<u8>, +) -> KResult<()> { let target = UserString::new(target)?; - let dentry = dentry_from(thread, dirfd, linkpath, false)?; + let dentry = dentry_from(thread, dirfd, linkpath, false).await?; - dentry.symlink(target.as_cstr().to_bytes()) + dentry.symlink(target.as_cstr().to_bytes()).await } #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_SYMLINK)] async fn symlink(target: User<u8>, linkpath: User<u8>) -> KResult<()> { - sys_symlinkat(thread, target, FD::AT_FDCWD, linkpath) + sys_symlinkat(thread, target, FD::AT_FDCWD, linkpath).await +} + +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +struct UserDeviceId(u32); + +impl FromSyscallArg for UserDeviceId { + fn from_arg(value: usize) -> Self { + Self(value as u32) + } +} + +impl UserDeviceId { + pub fn into_devid(self) -> DeviceId { + let major = (self.0 >> 8) & 0xfff; + let minor = (self.0 & 0xff) | ((self.0 >> 12) & 0xfff00); + + // TODO: We strip off the high 4 bits of the minor ID for now... + DeviceId::new(major as u16, minor as u16) + } } #[eonix_macros::define_syscall(SYS_MKNODAT)] -async fn mknodat(dirfd: FD, pathname: User<u8>, mut mode: Mode, dev: u32) -> KResult<()> { +async fn mknodat( + dirfd: FD, + pathname: User<u8>, + mut mode: Mode, + dev: UserDeviceId, +) -> KResult<()> { if !mode.is_blk() && !mode.is_chr() { return Err(EINVAL); } - let dentry = dentry_from(thread, dirfd, pathname, true)?; - - let umask = *thread.fs_context.umask.lock(); - mode.mask_perm(!umask.non_format_bits()); + let dentry = dentry_from(thread, dirfd, pathname, true).await?; + mode.set_perm(mode.perm().mask_with(*thread.fs_context.umask.lock())); - dentry.mknod(mode, dev) + dentry.mknod(mode, dev.into_devid()).await } #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_MKNOD)] -async fn mknod(pathname: User<u8>, mode: u32, dev: u32) -> KResult<()> { +async fn mknod( + pathname: User<u8>, + mode: Mode, + dev: UserDeviceId, +) -> KResult<()> { sys_mknodat(thread, FD::AT_FDCWD, pathname, mode, dev).await } @@ -340,19 +411,28 @@ async fn readlinkat( buffer: UserMut<u8>, bufsize: usize, ) -> KResult<usize> { - let dentry = dentry_from(thread, dirfd, pathname, false)?; + let dentry = dentry_from(thread, dirfd, pathname, false).await?; let mut buffer = UserBuffer::new(buffer, bufsize)?; - dentry.readlink(&mut buffer) + dentry.readlink(&mut buffer).await } #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_READLINK)] -async fn readlink(pathname: User<u8>, buffer: UserMut<u8>, bufsize: usize) -> KResult<usize> { +async fn readlink( + pathname: User<u8>, + buffer: UserMut<u8>, + bufsize: usize, +) -> KResult<usize> { sys_readlinkat(thread, FD::AT_FDCWD, pathname, buffer, bufsize).await } -async fn do_lseek(thread: &Thread, fd: FD, offset: u64, whence: u32) -> KResult<u64> { +async fn do_lseek( + thread: &Thread, + fd: FD, + offset: u64, + whence: u32, +) -> KResult<u64> { let file = thread.files.get(fd).ok_or(EBADF)?; Ok(match whence { @@ -371,14 +451,15 @@ async fn lseek(fd: FD, offset: u64, whence: u32) -> KResult<u64> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_LLSEEK)] -fn llseek( +async fn llseek( fd: FD, offset_high: u32, offset_low: u32, result: UserMut<u64>, whence: u32, ) -> KResult<()> { - let mut result = UserBuffer::new(result.cast(), core::mem::size_of::<u64>())?; + let mut result = + UserBuffer::new(result.cast(), core::mem::size_of::<u64>())?; let offset = ((offset_high as u64) << 32) | (offset_low as u64); let new_offset = do_lseek(thread, fd, offset, whence).await?; @@ -409,9 +490,10 @@ async fn readv(fd: FD, iov_user: User<IoVec>, iovcnt: u32) -> KResult<usize> { Ok(IoVec { len: Long::ZERO, .. }) => None, - Ok(IoVec { base, len }) => { - Some(UserBuffer::new(UserMut::with_addr(base.addr()), len.get())) - } + Ok(IoVec { base, len }) => Some(UserBuffer::new( + UserMut::with_addr(base.addr()), + len.get(), + )), }) .collect::<KResult<Vec<_>>>()?; @@ -446,8 +528,11 @@ async fn writev(fd: FD, iov_user: User<IoVec>, iovcnt: u32) -> KResult<usize> { len: Long::ZERO, .. }) => None, Ok(IoVec { base, len }) => Some( - CheckedUserPointer::new(User::with_addr(base.addr()), len.get()) - .map(|ptr| ptr.into_stream()), + CheckedUserPointer::new( + User::with_addr(base.addr()), + len.get(), + ) + .map(|ptr| ptr.into_stream()), ), }) .collect::<KResult<Vec<_>>>()?; @@ -466,12 +551,17 @@ async fn writev(fd: FD, iov_user: User<IoVec>, iovcnt: u32) -> KResult<usize> { } #[eonix_macros::define_syscall(SYS_FACCESSAT)] -async fn faccessat(dirfd: FD, pathname: User<u8>, _mode: u32, flags: AtFlags) -> KResult<()> { +async fn faccessat( + dirfd: FD, + pathname: User<u8>, + _mode: u32, + flags: AtFlags, +) -> KResult<()> { let dentry = if flags.at_empty_path() { let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; if !dentry.is_valid() { @@ -497,7 +587,12 @@ async fn access(pathname: User<u8>, mode: u32) -> KResult<()> { } #[eonix_macros::define_syscall(SYS_SENDFILE64)] -async fn sendfile64(out_fd: FD, in_fd: FD, offset: UserMut<u8>, count: usize) -> KResult<usize> { +async fn sendfile64( + out_fd: FD, + in_fd: FD, + offset: UserMut<u8>, + count: usize, +) -> KResult<usize> { let in_file = thread.files.get(in_fd).ok_or(EBADF)?; let out_file = thread.files.get(out_fd).ok_or(EBADF)?; @@ -602,7 +697,11 @@ async fn pselect6( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_POLL)] -async fn poll(fds: UserMut<UserPollFd>, nfds: u32, timeout: u32) -> KResult<u32> { +async fn poll( + fds: UserMut<UserPollFd>, + nfds: u32, + timeout: u32, +) -> KResult<u32> { do_poll(thread, fds, nfds, timeout).await } @@ -614,28 +713,34 @@ async fn fchownat( gid: u32, flags: AtFlags, ) -> KResult<()> { - let dentry = dentry_from(thread, dirfd, pathname, !flags.no_follow())?; + let dentry = + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await?; if !dentry.is_valid() { return Err(ENOENT); } - dentry.chown(uid, gid) + dentry.chown(uid, gid).await } #[eonix_macros::define_syscall(SYS_FCHMODAT)] -async fn fchmodat(dirfd: FD, pathname: User<u8>, mode: Mode, flags: AtFlags) -> KResult<()> { +async fn fchmodat( + dirfd: FD, + pathname: User<u8>, + mode: Mode, + flags: AtFlags, +) -> KResult<()> { let dentry = if flags.at_empty_path() { let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; if !dentry.is_valid() { return Err(ENOENT); } - dentry.chmod(mode) + dentry.chmod(mode).await } #[eonix_macros::define_syscall(SYS_FCHMOD)] @@ -654,7 +759,7 @@ async fn utimensat( let file = thread.files.get(dirfd).ok_or(EBADF)?; file.as_path().ok_or(EBADF)?.clone() } else { - dentry_from(thread, dirfd, pathname, !flags.no_follow())? + dentry_from(thread, dirfd, pathname, !flags.no_follow()).await? }; if !dentry.is_valid() { @@ -684,14 +789,18 @@ async fn renameat2( let flags = RenameFlags::from_bits(flags).ok_or(EINVAL)?; // The two flags RENAME_NOREPLACE and RENAME_EXCHANGE are mutually exclusive. - if flags.contains(RenameFlags::RENAME_NOREPLACE | RenameFlags::RENAME_EXCHANGE) { + if flags + .contains(RenameFlags::RENAME_NOREPLACE | RenameFlags::RENAME_EXCHANGE) + { Err(EINVAL)?; } - let old_dentry = dentry_from(thread, old_dirfd, old_pathname, false)?; - let new_dentry = dentry_from(thread, new_dirfd, new_pathname, false)?; + let old_dentry = + dentry_from(thread, old_dirfd, old_pathname, false).await?; + let new_dentry = + dentry_from(thread, new_dirfd, new_pathname, false).await?; - old_dentry.rename(&new_dentry, flags) + old_dentry.rename(&new_dentry, flags).await } #[cfg(target_arch = "x86_64")] diff --git a/src/kernel/syscall/mm.rs b/src/kernel/syscall/mm.rs index c6300ac7..825440ef 100644 --- a/src/kernel/syscall/mm.rs +++ b/src/kernel/syscall/mm.rs @@ -1,22 +1,15 @@ -use super::FromSyscallArg; -use crate::fs::shm::{gen_shm_id, ShmFlags, IPC_PRIVATE, SHM_MANAGER}; -use crate::kernel::constants::{EBADF, EEXIST, EINVAL, ENOENT}; -use crate::kernel::mem::FileMapping; -use crate::kernel::task::Thread; -use crate::kernel::vfs::filearray::FD; -use crate::kernel::vfs::inode::Mode; -use crate::{ - kernel::{ - constants::{UserMmapFlags, UserMmapProtocol}, - mem::{Mapping, Permission}, - }, - prelude::*, -}; use align_ext::AlignExt; use eonix_mm::address::{Addr as _, AddrOps as _, VAddr}; use eonix_mm::paging::PAGE_SIZE; use posix_types::syscall_no::*; +use super::FromSyscallArg; +use crate::kernel::constants::{UserMmapFlags, UserMmapProtocol, EBADF, EINVAL}; +use crate::kernel::mem::{FileMapping, Mapping, Permission}; +use crate::kernel::task::Thread; +use crate::kernel::vfs::filearray::FD; +use crate::prelude::*; + impl FromSyscallArg for UserMmapProtocol { fn from_arg(value: usize) -> UserMmapProtocol { UserMmapProtocol::from_bits_truncate(value as u32) @@ -66,13 +59,7 @@ async fn do_mmap2( if !is_shared { Mapping::Anonymous } else { - // The mode is unimportant here, since we are checking prot in mm_area. - let shared_area = SHM_MANAGER.lock().await.create_shared_area( - len, - thread.process.pid, - Mode::REG.perm(0o777), - ); - Mapping::File(FileMapping::new(shared_area.area.clone(), 0, len)) + unimplemented!("mmap MAP_ANONYMOUS | MAP_SHARED"); } } else { let file = thread @@ -82,7 +69,7 @@ async fn do_mmap2( .get_inode()? .ok_or(EBADF)?; - Mapping::File(FileMapping::new(file, pgoffset, len)) + Mapping::File(FileMapping::new(file.get_page_cache(), pgoffset, len)) }; let permission = Permission { @@ -179,114 +166,4 @@ async fn mprotect(addr: usize, len: usize, prot: UserMmapProtocol) -> KResult<() .await } -#[eonix_macros::define_syscall(SYS_SHMGET)] -async fn shmget(key: usize, size: usize, shmflg: u32) -> KResult<u32> { - let size = size.align_up(PAGE_SIZE); - - let mut shm_manager = SHM_MANAGER.lock().await; - let shmid = gen_shm_id(key)?; - - let mode = Mode::REG.perm(shmflg); - let shmflg = ShmFlags::from_bits_truncate(shmflg); - - if key == IPC_PRIVATE { - let new_shm = shm_manager.create_shared_area(size, thread.process.pid, mode); - shm_manager.insert(shmid, new_shm); - return Ok(shmid); - } - - if let Some(_) = shm_manager.get(shmid) { - if shmflg.contains(ShmFlags::IPC_CREAT | ShmFlags::IPC_EXCL) { - return Err(EEXIST); - } - - return Ok(shmid); - } - - if shmflg.contains(ShmFlags::IPC_CREAT) { - let new_shm = shm_manager.create_shared_area(size, thread.process.pid, mode); - shm_manager.insert(shmid, new_shm); - return Ok(shmid); - } - - Err(ENOENT) -} - -#[eonix_macros::define_syscall(SYS_SHMAT)] -async fn shmat(shmid: u32, addr: usize, shmflg: u32) -> KResult<usize> { - let mm_list = &thread.process.mm_list; - let shm_manager = SHM_MANAGER.lock().await; - let shm_area = shm_manager.get(shmid).ok_or(EINVAL)?; - - // Why is this not used? - let _mode = shmflg & 0o777; - let shmflg = ShmFlags::from_bits_truncate(shmflg); - - let mut permission = Permission { - read: true, - write: true, - execute: false, - }; - - if shmflg.contains(ShmFlags::SHM_EXEC) { - permission.execute = true; - } - if shmflg.contains(ShmFlags::SHM_RDONLY) { - permission.write = false; - } - - let size = shm_area.shmid_ds.shm_segsz; - - let mapping = Mapping::File(FileMapping { - file: shm_area.area.clone(), - offset: 0, - length: size, - }); - - let addr = if addr != 0 { - if addr % PAGE_SIZE != 0 && !shmflg.contains(ShmFlags::SHM_RND) { - return Err(EINVAL); - } - let addr = VAddr::from(addr.align_down(PAGE_SIZE)); - mm_list - .mmap_fixed(addr, size, mapping, permission, true) - .await - } else { - mm_list - .mmap_hint(VAddr::NULL, size, mapping, permission, true) - .await - }?; - - thread.process.shm_areas.lock().insert(addr, size); - - Ok(addr.addr()) -} - -#[eonix_macros::define_syscall(SYS_SHMDT)] -async fn shmdt(addr: usize) -> KResult<()> { - let addr = VAddr::from(addr); - - let size = { - let mut shm_areas = thread.process.shm_areas.lock(); - let size = *shm_areas.get(&addr).ok_or(EINVAL)?; - shm_areas.remove(&addr); - - size - }; - - thread.process.mm_list.unmap(addr, size).await -} - -#[eonix_macros::define_syscall(SYS_SHMCTL)] -async fn shmctl(_shmid: u32, _op: i32, _shmid_ds: usize) -> KResult<usize> { - // TODO - Ok(0) -} - -#[eonix_macros::define_syscall(SYS_MEMBARRIER)] -async fn membarrier(_cmd: usize, _flags: usize) -> KResult<()> { - // TODO - Ok(()) -} - pub fn keep_alive() {} diff --git a/src/kernel/syscall/procops.rs b/src/kernel/syscall/procops.rs index 7dd573cc..72d09f98 100644 --- a/src/kernel/syscall/procops.rs +++ b/src/kernel/syscall/procops.rs @@ -1,38 +1,42 @@ +use alloc::borrow::ToOwned; +use alloc::ffi::CString; +use core::time::Duration; + +use bitflags::bitflags; +use eonix_hal::traits::trap::RawTrapContext; +use eonix_hal::trap::TrapContext; +use eonix_log::println_warn; +use eonix_mm::address::Addr as _; +use eonix_sync::AsProof as _; +use posix_types::ctypes::PtrT; +use posix_types::signal::{SigAction, SigInfo, SigSet, Signal}; +use posix_types::stat::{TimeSpec, TimeVal}; +use posix_types::syscall_no::*; +use posix_types::SIGNAL_NOW; + use super::SyscallNoReturn; use crate::io::Buffer; use crate::kernel::constants::{ - CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, EINVAL, ENOENT, ENOTDIR, ERANGE, ESRCH, -}; -use crate::kernel::constants::{ - ENOSYS, PR_GET_NAME, PR_SET_NAME, RLIMIT_STACK, SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, + CLOCK_MONOTONIC, CLOCK_REALTIME, CLOCK_REALTIME_COARSE, EINVAL, ENOENT, + ENOSYS, ENOTDIR, ERANGE, ESRCH, PR_GET_NAME, PR_SET_NAME, RLIMIT_STACK, + SIG_BLOCK, SIG_SETMASK, SIG_UNBLOCK, }; use crate::kernel::mem::PageBuffer; use crate::kernel::syscall::{User, UserMut}; use crate::kernel::task::{ - do_clone, futex_wait, futex_wake, yield_now, FutexFlags, FutexOp, ProcessList, ProgramLoader, - RobustListHead, SignalAction, Thread, WaitId, WaitType, + do_clone, futex_exec, futex_wait, futex_wake, parse_futexop, yield_now, + CloneArgs, FutexFlags, FutexOp, ProcessList, ProgramLoader, RobustListHead, + SignalAction, Thread, WaitId, WaitType, }; -use crate::kernel::task::{parse_futexop, CloneArgs}; use crate::kernel::timer::sleep; -use crate::kernel::user::UserString; -use crate::kernel::user::{UserPointer, UserPointerMut}; -use crate::kernel::vfs::inode::Mode; -use crate::kernel::vfs::{self, dentry::Dentry}; +use crate::kernel::user::{ + UserBuffer, UserPointer, UserPointerMut, UserString, +}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::types::Permission; +use crate::kernel::vfs::{self}; use crate::path::Path; -use crate::{kernel::user::UserBuffer, prelude::*}; -use alloc::borrow::ToOwned; -use alloc::ffi::CString; -use bitflags::bitflags; -use core::time::Duration; -use eonix_hal::processor::UserTLS; -use eonix_hal::traits::trap::RawTrapContext; -use eonix_hal::trap::TrapContext; -use eonix_mm::address::Addr as _; -use eonix_sync::AsProof as _; -use posix_types::ctypes::PtrT; -use posix_types::signal::{SigAction, SigInfo, SigSet, Signal}; -use posix_types::stat::TimeVal; -use posix_types::{syscall_no::*, SIGNAL_NOW}; +use crate::prelude::*; #[repr(C)] #[derive(Debug, Clone, Copy)] @@ -50,7 +54,9 @@ bitflags! { } #[eonix_macros::define_syscall(SYS_NANOSLEEP)] -async fn nanosleep(req: User<(u32, u32)>, rem: UserMut<(u32, u32)>) -> KResult<usize> { +async fn nanosleep( + req: User<(u32, u32)>, rem: UserMut<(u32, u32)>, +) -> KResult<usize> { let req = UserPointer::new(req)?.read()?; let rem = if rem.is_null() { None @@ -58,7 +64,8 @@ async fn nanosleep(req: User<(u32, u32)>, rem: UserMut<(u32, u32)>) -> KResult<u Some(UserPointerMut::new(rem)?) }; - let duration = Duration::from_secs(req.0 as u64) + Duration::from_nanos(req.1 as u64); + let duration = + Duration::from_secs(req.0 as u64) + Duration::from_nanos(req.1 as u64); sleep(duration).await; if let Some(rem) = rem { @@ -70,10 +77,7 @@ async fn nanosleep(req: User<(u32, u32)>, rem: UserMut<(u32, u32)>) -> KResult<u #[eonix_macros::define_syscall(SYS_CLOCK_NANOSLEEP)] async fn clock_nanosleep( - clock_id: u32, - _flags: u32, - req: User<(u32, u32)>, - rem: UserMut<(u32, u32)>, + clock_id: u32, _flags: u32, req: User<(u32, u32)>, rem: UserMut<(u32, u32)>, ) -> KResult<usize> { if clock_id != CLOCK_REALTIME && clock_id != CLOCK_REALTIME_COARSE @@ -89,7 +93,8 @@ async fn clock_nanosleep( Some(UserPointerMut::new(rem)?) }; - let duration = Duration::from_secs(req.0 as u64) + Duration::from_nanos(req.1 as u64); + let duration = + Duration::from_secs(req.0 as u64) + Duration::from_nanos(req.1 as u64); sleep(duration).await; if let Some(rem) = rem { @@ -100,10 +105,12 @@ async fn clock_nanosleep( } #[eonix_macros::define_syscall(SYS_UMASK)] -async fn umask(mask: Mode) -> KResult<Mode> { - let mut umask = thread.fs_context.umask.lock(); +async fn umask(raw_new_mask: u32) -> KResult<u32> { + let new_mask = Permission::new(!raw_new_mask); + let old_mask = + core::mem::replace(&mut *thread.fs_context.umask.lock(), new_mask); - Ok(core::mem::replace(&mut *umask, mask.non_format())) + Ok(!old_mask.bits()) } #[eonix_macros::define_syscall(SYS_GETCWD)] @@ -124,7 +131,7 @@ async fn chdir(path: User<u8>) -> KResult<()> { let path = UserString::new(path)?; let path = Path::new(path.as_cstr().to_bytes())?; - let dentry = Dentry::open(&thread.fs_context, path, true)?; + let dentry = Dentry::open(&thread.fs_context, path, true).await?; if !dentry.is_valid() { return Err(ENOENT); } @@ -147,7 +154,9 @@ async fn umount(source: User<u8>) -> KResult<()> { } #[eonix_macros::define_syscall(SYS_MOUNT)] -async fn mount(source: User<u8>, target: User<u8>, fstype: User<u8>, flags: usize) -> KResult<()> { +async fn mount( + source: User<u8>, target: User<u8>, fstype: User<u8>, flags: usize, +) -> KResult<()> { let source = UserString::new(source)?; if source.as_cstr().to_str().unwrap() == "/dev/vda2" { return Ok(()); @@ -159,7 +168,8 @@ async fn mount(source: User<u8>, target: User<u8>, fstype: User<u8>, flags: usiz &thread.fs_context, Path::new(target.as_cstr().to_bytes())?, true, - )?; + ) + .await?; if !mountpoint.is_valid() { return Err(ENOENT); @@ -172,9 +182,12 @@ async fn mount(source: User<u8>, target: User<u8>, fstype: User<u8>, flags: usiz fstype.as_cstr().to_str().map_err(|_| EINVAL)?, flags as u64, ) + .await } -fn get_strings(mut ptr_strings: UserPointer<'_, PtrT>) -> KResult<Vec<CString>> { +fn get_strings( + mut ptr_strings: UserPointer<'_, PtrT>, +) -> KResult<Vec<CString>> { let mut strings = Vec::new(); loop { @@ -192,28 +205,36 @@ fn get_strings(mut ptr_strings: UserPointer<'_, PtrT>) -> KResult<Vec<CString>> } #[eonix_macros::define_syscall(SYS_EXECVE)] -async fn execve(exec: User<u8>, argv: User<PtrT>, envp: User<PtrT>) -> KResult<SyscallNoReturn> { +async fn execve( + exec: User<u8>, argv: User<PtrT>, envp: User<PtrT>, +) -> KResult<SyscallNoReturn> { let exec = UserString::new(exec)?; let exec = exec.as_cstr().to_owned(); let argv = get_strings(UserPointer::new(argv)?)?; let envp = get_strings(UserPointer::new(envp)?)?; - let dentry = Dentry::open(&thread.fs_context, Path::new(exec.as_bytes())?, true)?; + let dentry = + Dentry::open(&thread.fs_context, Path::new(exec.as_bytes())?, true) + .await?; if !dentry.is_valid() { Err(ENOENT)?; } // TODO: When `execve` is called by one of the threads in a process, the other threads // should be terminated and `execve` is performed in the thread group leader. - let load_info = ProgramLoader::parse(&thread.fs_context, exec, dentry.clone(), argv, envp)? - .load() - .await?; + let load_info = ProgramLoader::parse( + &thread.fs_context, + exec, + dentry.clone(), + argv, + envp, + ) + .await? + .load() + .await?; - if let Some(robust_list) = thread.get_robust_list() { - let _ = robust_list.wake_all().await; - thread.set_robust_list(None); - } + futex_exec(thread).await; unsafe { // SAFETY: We are doing execve, all other threads are terminated. @@ -225,10 +246,8 @@ async fn execve(exec: User<u8>, argv: User<PtrT>, envp: User<PtrT>) -> KResult<S thread.set_name(dentry.get_name()); let mut trap_ctx = thread.trap_ctx.borrow(); - *trap_ctx = TrapContext::new(); + *trap_ctx = TrapContext::new(true, true); - trap_ctx.set_user_mode(true); - trap_ctx.set_interrupt_enabled(true); trap_ctx.set_program_counter(load_info.entry_ip.addr()); trap_ctx.set_stack_pointer(load_info.sp.addr()); @@ -237,24 +256,15 @@ async fn execve(exec: User<u8>, argv: User<PtrT>, envp: User<PtrT>) -> KResult<S #[eonix_macros::define_syscall(SYS_EXIT)] async fn exit(status: u32) -> SyscallNoReturn { - let mut procs = ProcessList::get().write().await; - - unsafe { - procs - .do_exit(&thread, WaitType::Exited(status), false) - .await; - } + thread.exit(WaitType::Exited(status)); SyscallNoReturn } #[eonix_macros::define_syscall(SYS_EXIT_GROUP)] async fn exit_group(status: u32) -> SyscallNoReturn { - let mut procs = ProcessList::get().write().await; - - unsafe { - procs.do_exit(&thread, WaitType::Exited(status), true).await; - } + // XXX: Send SIGKILL to our sibling threads. + thread.exit(WaitType::Exited(status)); SyscallNoReturn } @@ -266,10 +276,7 @@ enum WaitInfo { } async fn do_waitid( - thread: &Thread, - wait_id: WaitId, - info: WaitInfo, - options: u32, + thread: &Thread, wait_id: WaitId, info: WaitInfo, options: u32, rusage: UserMut<RUsage>, ) -> KResult<u32> { if !rusage.is_null() { @@ -309,7 +316,8 @@ async fn do_waitid( Ok(0) } WaitInfo::Status(status_ptr) => { - UserPointerMut::new(status_ptr)?.write(wait_object.code.to_wstatus())?; + UserPointerMut::new(status_ptr)? + .write(wait_object.code.to_wstatus())?; Ok(wait_object.pid) } WaitInfo::None => Ok(wait_object.pid), @@ -318,10 +326,7 @@ async fn do_waitid( #[eonix_macros::define_syscall(SYS_WAITID)] async fn waitid( - id_type: u32, - id: u32, - info: UserMut<SigInfo>, - options: u32, + id_type: u32, id: u32, info: UserMut<SigInfo>, options: u32, rusage: UserMut<RUsage>, ) -> KResult<u32> { let wait_id = WaitId::from_type_and_id(id_type, id)?; @@ -343,10 +348,7 @@ async fn waitid( #[eonix_macros::define_syscall(SYS_WAIT4)] async fn wait4( - wait_id: i32, - arg1: UserMut<u32>, - options: u32, - rusage: UserMut<RUsage>, + wait_id: i32, arg1: UserMut<u32>, options: u32, rusage: UserMut<RUsage>, ) -> KResult<u32> { let waitinfo = if arg1.is_null() { WaitInfo::None @@ -361,8 +363,10 @@ async fn wait4( #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_WAITPID)] -async fn waitpid(waitpid: i32, arg1: UserMut<u32>, options: u32) -> KResult<u32> { - sys_wait4(thread, waitpid, arg1, options, core::ptr::null_mut()).await +async fn waitpid( + waitpid: i32, arg1: UserMut<u32>, options: u32, +) -> KResult<u32> { + sys_wait4(thread, waitpid, arg1, options, UserMut::null()).await } #[eonix_macros::define_syscall(SYS_SETSID)] @@ -489,51 +493,15 @@ async fn gettid() -> KResult<u32> { Ok(thread.tid) } -pub fn parse_user_tls(arch_tls: usize) -> KResult<UserTLS> { - #[cfg(target_arch = "x86_64")] - { - let desc = arch_tls as *mut posix_types::x86_64::UserDescriptor; - let desc_pointer = UserPointerMut::new(desc)?; - let mut desc = desc_pointer.read()?; - - // Clear the TLS area if it is not present. - if desc.flags.is_read_exec_only() && !desc.flags.is_present() { - if desc.limit != 0 && desc.base != 0 { - let len = if desc.flags.is_limit_in_pages() { - (desc.limit as usize) << 12 - } else { - desc.limit as usize - }; - - CheckedUserPointer::new(desc.base as _, len)?.zero()?; - } - } - - let (new_tls, entry) = - UserTLS::new32(desc.base, desc.limit, desc.flags.is_limit_in_pages()); - desc.entry = entry; - desc_pointer.write(desc)?; - - Ok(new_tls) - } - - #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] - { - Ok(UserTLS::new(arch_tls as u64)) - } -} - #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_SET_THREAD_AREA)] -async fn set_thread_area(arch_tls: usize) -> KResult<()> { - thread.set_user_tls(parse_user_tls(arch_tls)?)?; +async fn set_thread_area(tls: PtrT) -> KResult<()> { + use crate::kernel::task::UserTLSDescriptor; - // SAFETY: Preemption is disabled on calling `load_thread_area32()`. - unsafe { - eonix_preempt::disable(); - thread.load_thread_area32(); - eonix_preempt::enable(); - } + let tls = UserTLSDescriptor::new(tls)?.read()?; + + thread.set_user_tls(tls)?; + thread.activate_tls(); Ok(()) } @@ -548,7 +516,9 @@ async fn set_tid_address(tidptr: UserMut<u32>) -> KResult<u32> { async fn prctl(option: u32, arg2: PtrT) -> KResult<()> { match option { PR_SET_NAME => { - let name = UserPointer::<[u8; 16]>::new(User::with_addr(arg2.addr()))?.read()?; + let name = + UserPointer::<[u8; 16]>::new(User::with_addr(arg2.addr()))? + .read()?; let len = name.iter().position(|&c| c == 0).unwrap_or(15); thread.set_name(name[..len].into()); Ok(()) @@ -556,8 +526,10 @@ async fn prctl(option: u32, arg2: PtrT) -> KResult<()> { PR_GET_NAME => { let name = thread.get_name(); let len = name.len().min(15); - let name: [u8; 16] = core::array::from_fn(|i| if i < len { name[i] } else { 0 }); - UserPointerMut::<[u8; 16]>::new(UserMut::with_addr(arg2.addr()))?.write(name)?; + let name: [u8; 16] = + core::array::from_fn(|i| if i < len { name[i] } else { 0 }); + UserPointerMut::<[u8; 16]>::new(UserMut::with_addr(arg2.addr()))? + .write(name)?; Ok(()) } _ => Err(EINVAL), @@ -617,10 +589,7 @@ async fn tgkill(tgid: u32, tid: u32, sig: u32) -> KResult<()> { #[eonix_macros::define_syscall(SYS_RT_SIGPROCMASK)] async fn rt_sigprocmask( - how: u32, - set: UserMut<SigSet>, - oldset: UserMut<SigSet>, - sigsetsize: usize, + how: u32, set: UserMut<SigSet>, oldset: UserMut<SigSet>, sigsetsize: usize, ) -> KResult<()> { if sigsetsize != size_of::<SigSet>() { return Err(EINVAL); @@ -647,18 +616,12 @@ async fn rt_sigprocmask( Ok(()) } -#[repr(C)] -#[derive(Clone, Copy)] -struct TimeSpec32 { - tv_sec: i32, - tv_nsec: i32, -} - -#[eonix_macros::define_syscall(SYS_RT_SIGTIMEDWAIT_TIME32)] -async fn rt_sigtimedwait_time32( - _uthese: User<SigSet>, - _uinfo: UserMut<SigInfo>, - _uts: User<TimeSpec32>, +#[cfg_attr( + any(target_arch = "riscv64", target_arch = "loongarch64"), + eonix_macros::define_syscall(SYS_RT_SIGTIMEDWAIT) +)] +async fn rt_sigtimedwait( + _uthese: User<SigSet>, _uinfo: UserMut<SigInfo>, _uts: User<TimeSpec>, ) -> KResult<i32> { // TODO Ok(0) @@ -666,9 +629,7 @@ async fn rt_sigtimedwait_time32( #[eonix_macros::define_syscall(SYS_RT_SIGACTION)] async fn rt_sigaction( - signum: u32, - act: User<SigAction>, - oldact: UserMut<SigAction>, + signum: u32, act: User<SigAction>, oldact: UserMut<SigAction>, sigsetsize: usize, ) -> KResult<()> { let signal = Signal::try_from_raw(signum)?; @@ -698,9 +659,7 @@ async fn rt_sigaction( #[eonix_macros::define_syscall(SYS_PRLIMIT64)] async fn prlimit64( - pid: u32, - resource: u32, - new_limit: User<RLimit>, + pid: u32, resource: u32, new_limit: User<RLimit>, old_limit: UserMut<RLimit>, ) -> KResult<()> { if pid != 0 { @@ -813,13 +772,16 @@ async fn fork() -> KResult<u32> { #[cfg(not(target_arch = "loongarch64"))] #[eonix_macros::define_syscall(SYS_CLONE)] async fn clone( - clone_flags: usize, - new_sp: usize, - parent_tidptr: UserMut<u32>, - tls: usize, + clone_flags: usize, new_sp: usize, parent_tidptr: UserMut<u32>, tls: PtrT, child_tidptr: UserMut<u32>, ) -> KResult<u32> { - let clone_args = CloneArgs::for_clone(clone_flags, new_sp, child_tidptr, parent_tidptr, tls)?; + let clone_args = CloneArgs::for_clone( + clone_flags, + new_sp, + child_tidptr, + parent_tidptr, + tls, + )?; do_clone(thread, clone_args).await } @@ -827,24 +789,23 @@ async fn clone( #[cfg(target_arch = "loongarch64")] #[eonix_macros::define_syscall(SYS_CLONE)] async fn clone( - clone_flags: usize, - new_sp: usize, - parent_tidptr: UserMut<u32>, - child_tidptr: UserMut<u32>, - tls: usize, + clone_flags: usize, new_sp: usize, parent_tidptr: UserMut<u32>, + child_tidptr: UserMut<u32>, tls: usize, ) -> KResult<u32> { - let clone_args = CloneArgs::for_clone(clone_flags, new_sp, child_tidptr, parent_tidptr, tls)?; + let clone_args = CloneArgs::for_clone( + clone_flags, + new_sp, + child_tidptr, + parent_tidptr, + tls, + )?; do_clone(thread, clone_args).await } #[eonix_macros::define_syscall(SYS_FUTEX)] async fn futex( - uaddr: usize, - op: u32, - val: u32, - _time_out: usize, - _uaddr2: usize, + uaddr: usize, op: u32, val: u32, _time_out: usize, _uaddr2: usize, _val3: u32, ) -> KResult<usize> { let (futex_op, futex_flag) = parse_futexop(op)?; @@ -873,7 +834,9 @@ async fn futex( } #[eonix_macros::define_syscall(SYS_SET_ROBUST_LIST)] -async fn set_robust_list(head: User<RobustListHead>, len: usize) -> KResult<()> { +async fn set_robust_list( + head: User<RobustListHead>, len: usize, +) -> KResult<()> { if len != size_of::<RobustListHead>() { return Err(EINVAL); } @@ -893,7 +856,7 @@ async fn rt_sigreturn() -> KResult<SyscallNoReturn> { "`rt_sigreturn` failed in thread {} with error {err}!", thread.tid ); - thread.force_kill(Signal::SIGSEGV).await; + thread.force_kill(Signal::SIGSEGV); return Err(err); } @@ -912,7 +875,7 @@ async fn sigreturn() -> KResult<SyscallNoReturn> { "`sigreturn` failed in thread {} with error {err}!", thread.tid ); - thread.force_kill(Signal::SIGSEGV).await; + thread.force_kill(Signal::SIGSEGV); return Err(err); } @@ -921,8 +884,27 @@ async fn sigreturn() -> KResult<SyscallNoReturn> { #[cfg(target_arch = "x86_64")] #[eonix_macros::define_syscall(SYS_ARCH_PRCTL)] -async fn arch_prctl(option: u32, addr: u32) -> KResult<u32> { - sys_arch_prctl(thread, option, addr).await +async fn arch_prctl(option: u32, addr: PtrT) -> KResult<u32> { + match option { + PR_SET_NAME => { + let name = + UserPointer::<[u8; 16]>::new(User::with_addr(addr.addr()))? + .read()?; + let len = name.iter().position(|&c| c == 0).unwrap_or(15); + thread.set_name(name[..len].into()); + Ok(0) + } + PR_GET_NAME => { + let name = thread.get_name(); + let len = name.len().min(15); + let name: [u8; 16] = + core::array::from_fn(|i| if i < len { name[i] } else { 0 }); + UserPointerMut::<[u8; 16]>::new(UserMut::with_addr(addr.addr()))? + .write(name)?; + Ok(0) + } + _ => Err(EINVAL), + } } pub fn keep_alive() {} diff --git a/src/kernel/task.rs b/src/kernel/task.rs index 2ef58069..dd391405 100644 --- a/src/kernel/task.rs +++ b/src/kernel/task.rs @@ -8,17 +8,25 @@ mod process_list; mod session; mod signal; mod thread; +mod user_tls; pub use clone::{do_clone, CloneArgs, CloneFlags}; -pub use futex::{futex_wait, futex_wake, parse_futexop, FutexFlags, FutexOp, RobustListHead}; +use eonix_hal::symbol_addr; +pub use futex::{ + futex_exec, futex_exit, futex_wait, futex_wake, parse_futexop, FutexFlags, + FutexOp, RobustListHead, +}; pub use kernel_stack::KernelStack; pub use loader::ProgramLoader; -pub use process::{alloc_pid, Process, ProcessBuilder, WaitId, WaitObject, WaitType}; +pub use process::{ + alloc_pid, Process, ProcessBuilder, WaitId, WaitObject, WaitType, +}; pub use process_group::ProcessGroup; pub use process_list::ProcessList; pub use session::Session; pub use signal::SignalAction; pub use thread::{yield_now, Thread, ThreadAlloc, ThreadBuilder}; +pub use user_tls::{UserTLS, UserTLSDescriptor}; fn do_block_on<F>(mut future: core::pin::Pin<&mut F>) -> F::Output where @@ -79,33 +87,31 @@ pub async fn stackful<F>(mut future: F) -> F::Output where F: core::future::Future, { - use crate::kernel::{ - interrupt::{default_fault_handler, default_irq_handler}, - timer::{should_reschedule, timer_interrupt}, - }; use alloc::sync::Arc; use alloc::task::Wake; use core::cell::UnsafeCell; use core::future::Future; use core::pin::Pin; use core::ptr::NonNull; - use core::sync::atomic::AtomicBool; - use core::sync::atomic::Ordering; - use core::task::Context; - use core::task::Poll; - use core::task::Waker; - use eonix_hal::traits::trap::RawTrapContext; - use eonix_hal::traits::trap::TrapReturn; - use eonix_hal::traits::trap::TrapType; + use core::sync::atomic::{AtomicBool, Ordering}; + use core::task::{Context, Poll, Waker}; + + use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; use eonix_hal::trap::TrapContext; use eonix_preempt::assert_preempt_enabled; - use eonix_runtime::executor::Stack; use eonix_runtime::task::Task; use thread::wait_for_wakeups; + use crate::kernel::interrupt::{ + default_fault_handler, default_irq_handler, + }; + use crate::kernel::timer::{should_reschedule, timer_interrupt}; + let stack = KernelStack::new(); - fn execute<F>(mut future: Pin<&mut F>, output_ptr: NonNull<Option<F::Output>>) -> ! + unsafe extern "C" fn execute<F>( + mut future: Pin<&mut F>, output_ptr: NonNull<Option<F::Output>>, + ) -> ! where F: Future, { @@ -138,7 +144,9 @@ where match future.as_mut().poll(&mut cx) { Poll::Ready(output) => break output, Poll::Pending => { - assert_preempt_enabled!("Blocking in stackful futures is not allowed."); + assert_preempt_enabled!( + "Blocking in stackful futures is not allowed." + ); if Task::current().is_ready() { continue; @@ -156,6 +164,9 @@ where #[cfg(target_arch = "loongarch64")] core::arch::asm!("break 1"); + + #[cfg(target_arch = "x86_64")] + core::arch::asm!("int 3"); } } } @@ -175,24 +186,23 @@ where #[cfg(target_arch = "loongarch64")] core::arch::asm!("break 1"); + + #[cfg(target_arch = "x86_64")] + core::arch::asm!("int 3"); } unreachable!() } - let sp = stack.get_bottom(); let mut output = UnsafeCell::new(None); - let mut trap_ctx = TrapContext::new(); + let mut trap_ctx = TrapContext::new(true, false); - trap_ctx.set_user_mode(false); - trap_ctx.set_interrupt_enabled(true); - let _ = trap_ctx.set_user_call_frame( - execute::<F> as usize, - Some(sp.addr().get()), + trap_ctx.set_kernel_call_frame( + symbol_addr!(execute::<F>), + &stack, None, &[(&raw mut future) as usize, output.get() as usize], - |_, _| Ok::<(), u32>(()), ); loop { @@ -202,7 +212,9 @@ where match trap_ctx.trap_type() { TrapType::Syscall { .. } => {} - TrapType::Fault(fault) => default_fault_handler(fault, &mut trap_ctx), + TrapType::Fault(fault) => { + default_fault_handler(fault, &mut trap_ctx) + } TrapType::Irq { callback } => callback(default_irq_handler), TrapType::Timer { callback } => { callback(timer_interrupt); @@ -219,10 +231,12 @@ where } #[cfg(target_arch = "riscv64")] - trap_ctx.set_program_counter(trap_ctx.get_program_counter() + 2); + trap_ctx + .set_program_counter(trap_ctx.get_program_counter() + 2); #[cfg(target_arch = "loongarch64")] - trap_ctx.set_program_counter(trap_ctx.get_program_counter() + 4); + trap_ctx + .set_program_counter(trap_ctx.get_program_counter() + 4); } } } diff --git a/src/kernel/task/clone.rs b/src/kernel/task/clone.rs index e0d578c1..392787de 100644 --- a/src/kernel/task/clone.rs +++ b/src/kernel/task/clone.rs @@ -1,18 +1,19 @@ -use crate::{ - kernel::{ - syscall::{procops::parse_user_tls, UserMut}, - task::{alloc_pid, ProcessBuilder, ProcessList, Thread, ThreadBuilder}, - user::UserPointerMut, - }, - KResult, -}; -use bitflags::bitflags; use core::num::NonZero; -use eonix_hal::processor::UserTLS; + +use bitflags::bitflags; use eonix_runtime::scheduler::RUNTIME; use eonix_sync::AsProof; +use posix_types::ctypes::PtrT; use posix_types::signal::Signal; +use super::{UserTLS, UserTLSDescriptor}; +use crate::kernel::syscall::UserMut; +use crate::kernel::task::{ + alloc_pid, ProcessBuilder, ProcessList, Thread, ThreadBuilder, +}; +use crate::kernel::user::UserPointerMut; +use crate::KResult; + bitflags! { #[derive(Debug, Default)] pub struct CloneFlags: usize { @@ -46,23 +47,26 @@ bitflags! { #[derive(Debug)] pub struct CloneArgs { pub flags: CloneFlags, - pub sp: Option<NonZero<usize>>, // Stack pointer for the new thread. - pub exit_signal: Option<Signal>, // Signal to send to the parent on exit. - pub set_tid_ptr: Option<UserMut<u32>>, // Pointer to set child TID in user space. - pub clear_tid_ptr: Option<UserMut<u32>>, // Pointer to clear child TID in user space. - pub parent_tid_ptr: Option<UserMut<u32>>, // Pointer to parent TID in user space. - pub tls: Option<UserTLS>, // Pointer to TLS information. + /// Stack pointer for the new thread. + pub sp: Option<NonZero<usize>>, + /// Signal to send to the parent on exit. + pub exit_signal: Option<Signal>, + /// Pointer to set child TID in user space. + pub set_tid_ptr: Option<UserMut<u32>>, + /// Pointer to clear child TID in user space. + pub clear_tid_ptr: Option<UserMut<u32>>, + /// Pointer to parent TID in user space. + pub parent_tid_ptr: Option<UserMut<u32>>, + /// Pointer to TLS information. + pub tls: Option<UserTLS>, } impl CloneArgs { const MASK: usize = 0xff; pub fn for_clone( - flags: usize, - sp: usize, - child_tid_ptr: UserMut<u32>, - parent_tid_ptr: UserMut<u32>, - tls: usize, + flags: usize, sp: usize, child_tid_ptr: UserMut<u32>, + parent_tid_ptr: UserMut<u32>, tls: PtrT, ) -> KResult<Self> { let clone_flags = CloneFlags::from_bits_truncate(flags & !Self::MASK); let exit_signal = flags & Self::MASK; @@ -87,7 +91,8 @@ impl CloneArgs { .then_some(parent_tid_ptr); let tls = if clone_flags.contains(CloneFlags::CLONE_SETTLS) { - Some(parse_user_tls(tls)?) + let tls_desc = UserTLSDescriptor::new(tls)?; + Some(tls_desc.read()?) } else { None }; @@ -133,7 +138,8 @@ impl CloneArgs { pub async fn do_clone(thread: &Thread, clone_args: CloneArgs) -> KResult<u32> { let mut procs = ProcessList::get().write().await; - let thread_builder = ThreadBuilder::new().clone_from(&thread, &clone_args)?; + let thread_builder = + ThreadBuilder::new().clone_from(&thread, &clone_args)?; let current_process = thread.process.clone(); let new_pid = alloc_pid(); @@ -142,7 +148,6 @@ pub async fn do_clone(thread: &Thread, clone_args: CloneArgs) -> KResult<u32> { let new_thread = thread_builder .process(current_process) .tid(new_pid) - .tls(clone_args.tls) .build(&mut procs); new_thread } else { diff --git a/src/kernel/task/futex.rs b/src/kernel/task/futex.rs index a04d7091..4dd57615 100644 --- a/src/kernel/task/futex.rs +++ b/src/kernel/task/futex.rs @@ -1,19 +1,17 @@ -use core::pin::pin; - use alloc::sync::Arc; use alloc::vec::Vec; +use core::pin::pin; + use bitflags::bitflags; +use eonix_mm::address::Addr; use eonix_sync::{LazyLock, Mutex, MutexGuard, WaitList}; use intrusive_collections::{intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink}; -use crate::{ - kernel::{ - constants::{EAGAIN, EINVAL}, - syscall::User, - user::UserPointer, - }, - prelude::KResult, -}; +use super::Thread; +use crate::kernel::constants::{EAGAIN, EINVAL}; +use crate::kernel::syscall::User; +use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::prelude::KResult; #[derive(PartialEq, Debug, Clone, Copy)] #[repr(u32)] @@ -318,3 +316,38 @@ impl RobustListHead { Ok(()) } } + +async fn do_futex_exit(thread: &Thread) -> KResult<()> { + if let Some(clear_ctid) = thread.get_clear_ctid() { + UserPointerMut::new(clear_ctid)?.write(0u32)?; + + futex_wake(clear_ctid.addr(), None, 1).await?; + } + + if let Some(robust_list) = thread.get_robust_list() { + robust_list.wake_all().await?; + } + + Ok(()) +} + +pub async fn futex_exit(thread: &Thread) { + // We don't care about any error happened inside. + // If they've set up a wrong pointer, good luck to them... + let _ = do_futex_exit(thread); +} + +async fn do_futex_exec(thread: &Thread) -> KResult<()> { + if let Some(robust_list) = thread.get_robust_list() { + robust_list.wake_all().await?; + thread.set_robust_list(None); + } + + Ok(()) +} + +pub async fn futex_exec(thread: &Thread) { + // We don't care about any error happened inside. + // If they've set up a wrong pointer, good luck to them... + let _ = do_futex_exec(thread); +} diff --git a/src/kernel/task/kernel_stack.rs b/src/kernel/task/kernel_stack.rs index d3e9de2f..30545552 100644 --- a/src/kernel/task/kernel_stack.rs +++ b/src/kernel/task/kernel_stack.rs @@ -1,11 +1,12 @@ -use crate::kernel::mem::{paging::Page, PhysAccess as _}; -use core::{num::NonZero, ptr::NonNull}; -use eonix_runtime::executor::Stack; +use core::ptr::NonNull; + +use eonix_hal::traits::trap::Stack; + +use crate::kernel::mem::FolioOwned; #[derive(Debug)] pub struct KernelStack { - _pages: Page, - bottom: NonZero<usize>, + folio: FolioOwned, } impl KernelStack { @@ -14,26 +15,22 @@ impl KernelStack { const KERNEL_STACK_ORDER: u32 = 7; pub fn new() -> Self { - let pages = Page::alloc_order(Self::KERNEL_STACK_ORDER); - let bottom = unsafe { - // SAFETY: The paddr is from a page, which should be valid. - pages.range().end().as_ptr::<u8>().addr() - }; - Self { - _pages: pages, - bottom, + folio: FolioOwned::alloc_order(Self::KERNEL_STACK_ORDER), } } -} -impl Stack for KernelStack { - fn new() -> Self { - Self::new() + pub fn get_bottom(&self) -> NonNull<()> { + let ptr = self.folio.get_bytes_ptr(); + let len = ptr.len(); + + // SAFETY: The vaddr of the folio is guaranteed to be non-zero. + unsafe { ptr.cast().byte_add(len) } } +} - fn get_bottom(&self) -> NonNull<()> { - // SAFETY: The stack is allocated and `bottom` is non-zero. - unsafe { NonNull::new_unchecked(self.bottom.get() as *mut _) } +impl Stack for KernelStack { + fn get_bottom(&self) -> *mut usize { + self.get_bottom().cast().as_ptr() } } diff --git a/src/kernel/task/loader/elf.rs b/src/kernel/task/loader/elf.rs index 859e0010..9f8aa166 100644 --- a/src/kernel/task/loader/elf.rs +++ b/src/kernel/task/loader/elf.rs @@ -1,27 +1,22 @@ +use alloc::ffi::CString; +use alloc::sync::Arc; +use alloc::vec::Vec; + +use align_ext::AlignExt; +use eonix_mm::address::{Addr, AddrOps as _, VAddr}; +use eonix_mm::paging::PAGE_SIZE; +use xmas_elf::header::{self, Class, HeaderPt1, Machine_}; +use xmas_elf::program::{self, ProgramHeader32, ProgramHeader64}; + use super::{LoadInfo, ELF_MAGIC}; -use crate::io::UninitBuffer; +use crate::io::{ByteBuffer, UninitBuffer}; +use crate::kernel::constants::ENOEXEC; +use crate::kernel::mem::{FileMapping, MMList, Mapping, Permission}; use crate::kernel::task::loader::aux_vec::{AuxKey, AuxVec}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::FsContext; use crate::path::Path; -use crate::{ - io::ByteBuffer, - kernel::{ - constants::ENOEXEC, - mem::{FileMapping, MMList, Mapping, Permission}, - vfs::{dentry::Dentry, FsContext}, - }, - prelude::*, -}; -use align_ext::AlignExt; -use alloc::vec::Vec; -use alloc::{ffi::CString, sync::Arc}; -use eonix_mm::{ - address::{Addr, AddrOps as _, VAddr}, - paging::PAGE_SIZE, -}; -use xmas_elf::{ - header::{self, Class, HeaderPt1, Machine_}, - program::{self, ProgramHeader32, ProgramHeader64}, -}; +use crate::prelude::*; const INIT_STACK_SIZE: usize = 0x80_0000; @@ -193,9 +188,9 @@ impl<E: ElfArch> Elf<E> { Err(ENOEXEC) } - fn parse(elf_file: Arc<Dentry>) -> KResult<Self> { + async fn parse(elf_file: Arc<Dentry>) -> KResult<Self> { let mut elf_header = UninitBuffer::<ElfHeader<E::Ea>>::new(); - elf_file.read(&mut elf_header, 0)?; + elf_file.read(&mut elf_header, 0).await?; let elf_header = elf_header.assume_init().map_err(|_| ENOEXEC)?; @@ -203,10 +198,12 @@ impl<E: ElfArch> Elf<E> { let ph_count = elf_header.pt2.ph_count; let mut program_headers = vec![E::Ph::default(); ph_count as usize]; - elf_file.read( - &mut ByteBuffer::from(program_headers.as_mut_slice()), - ph_offset.into_usize(), - )?; + elf_file + .read( + &mut ByteBuffer::from(program_headers.as_mut_slice()), + ph_offset.into_usize(), + ) + .await?; Ok(Self { file: elf_file, @@ -364,7 +361,7 @@ impl<E: ElfArch> Elf<E> { vmap_start, file_len, Mapping::File(FileMapping::new( - self.file.get_inode()?, + self.file.get_inode()?.get_page_cache(), file_offset, real_file_length, )), @@ -374,28 +371,40 @@ impl<E: ElfArch> Elf<E> { .await?; } - if vmem_len > file_len { - mm_list - .mmap_fixed( - vmap_start + file_len, - vmem_len - file_len, - Mapping::Anonymous, - permission, - false, - ) - .await?; + if vmem_vaddr_end > load_vaddr_end { + if load_vaddr_end.page_offset() != 0 { + let mut zero_len = PAGE_SIZE - load_vaddr_end.page_offset(); + zero_len = zero_len.min(vmem_vaddr_end - load_vaddr_end); + + mm_list + .access_mut(load_vaddr_end, zero_len, |_, data| data.fill(0)) + .await?; + } + + if vmem_len - file_len > 0 { + mm_list + .mmap_fixed( + vmap_start + file_len, + vmem_len - file_len, + Mapping::Anonymous, + permission, + false, + ) + .await?; + } } Ok(vmap_start + vmem_len) } async fn load_ldso(&self, mm_list: &MMList) -> KResult<Option<LdsoLoadInfo>> { - let ldso_path = self.ldso_path()?; + let ldso_path = self.ldso_path().await?; if let Some(ldso_path) = ldso_path { let fs_context = FsContext::global(); - let ldso_file = Dentry::open(fs_context, Path::new(ldso_path.as_bytes())?, true)?; - let ldso_elf = Elf::<E>::parse(ldso_file)?; + let ldso_file = + Dentry::open(fs_context, Path::new(ldso_path.as_bytes())?, true).await?; + let ldso_elf = Elf::<E>::parse(ldso_file).await?; let base = VAddr::from(E::LDSO_BASE_ADDR); @@ -420,7 +429,7 @@ impl<E: ElfArch> Elf<E> { mm_list.map_vdso().await } - fn ldso_path(&self) -> KResult<Option<String>> { + async fn ldso_path(&self) -> KResult<Option<String>> { for program_header in &self.program_headers { let type_ = program_header.type_().map_err(|_| ENOEXEC)?; @@ -430,7 +439,8 @@ impl<E: ElfArch> Elf<E> { let mut ldso_vec = vec![0u8; file_size - 1]; // -1 due to '\0' self.file - .read(&mut ByteBuffer::from(ldso_vec.as_mut_slice()), file_offset)?; + .read(&mut ByteBuffer::from(ldso_vec.as_mut_slice()), file_offset) + .await?; let ldso_path = String::from_utf8(ldso_vec).map_err(|_| ENOEXEC)?; return Ok(Some(ldso_path)); } @@ -445,16 +455,16 @@ pub enum ELF { } impl ELF { - pub fn parse(elf_file: Arc<Dentry>) -> KResult<Self> { + pub async fn parse(elf_file: Arc<Dentry>) -> KResult<Self> { let mut header_pt1 = UninitBuffer::<HeaderPt1>::new(); - elf_file.read(&mut header_pt1, 0)?; + elf_file.read(&mut header_pt1, 0).await?; let header_pt1 = header_pt1.assume_init().map_err(|_| ENOEXEC)?; assert_eq!(header_pt1.magic, ELF_MAGIC); match header_pt1.class() { - Class::ThirtyTwo => Ok(ELF::Elf32(Elf::parse(elf_file)?)), - Class::SixtyFour => Ok(ELF::Elf64(Elf::parse(elf_file)?)), + Class::ThirtyTwo => Ok(ELF::Elf32(Elf::parse(elf_file).await?)), + Class::SixtyFour => Ok(ELF::Elf64(Elf::parse(elf_file).await?)), _ => Err(ENOEXEC), } } diff --git a/src/kernel/task/loader/mod.rs b/src/kernel/task/loader/mod.rs index 4e3f4db1..7679aaf4 100644 --- a/src/kernel/task/loader/mod.rs +++ b/src/kernel/task/loader/mod.rs @@ -33,7 +33,7 @@ pub struct ProgramLoader { } impl ProgramLoader { - pub fn parse( + pub async fn parse( fs_context: &FsContext, mut exec_path: CString, mut file: Arc<Dentry>, @@ -49,12 +49,15 @@ impl ProgramLoader { } let mut magic = [0; 4]; - file.read(&mut ByteBuffer::new(magic.as_mut_slice()), 0)?; + file.read(&mut ByteBuffer::new(magic.as_mut_slice()), 0) + .await?; match magic { [b'#', b'!', ..] => { let mut interpreter_line = [0; 256]; - let nread = file.read(&mut ByteBuffer::new(&mut interpreter_line), 0)?; + let nread = file + .read(&mut ByteBuffer::new(&mut interpreter_line), 0) + .await?; // There is a tiny time gap between reading the magic number and // reading the interpreter line, so we need to check if the line @@ -77,7 +80,7 @@ impl ProgramLoader { } let path = Path::new(interpreter_name.as_bytes())?; - file = Dentry::open(fs_context, path, true)?; + file = Dentry::open(fs_context, path, true).await?; args.insert(0, interpreter_name.clone()); if let Some(arg) = interpreter_arg { @@ -92,7 +95,7 @@ impl ProgramLoader { exec_path = interpreter_name; } - ELF_MAGIC => break ELF::parse(file)?, + ELF_MAGIC => break ELF::parse(file).await?, _ => return Err(ENOEXEC), } diff --git a/src/kernel/task/process.rs b/src/kernel/task/process.rs index 421e4b8b..6299f87f 100644 --- a/src/kernel/task/process.rs +++ b/src/kernel/task/process.rs @@ -1,34 +1,32 @@ -use super::{ - process_group::ProcessGroupBuilder, signal::RaiseResult, thread::ThreadBuilder, ProcessGroup, - ProcessList, Session, Thread, -}; -use crate::kernel::constants::{ECHILD, EINTR, EINVAL, EPERM, ESRCH}; -use crate::kernel::task::{CloneArgs, CloneFlags}; -use crate::rcu::call_rcu; -use crate::{ - kernel::mem::MMList, - prelude::*, - rcu::{RCUPointer, RCUReadGuard}, - sync::CondVar, -}; -use alloc::{ - collections::{btree_map::BTreeMap, vec_deque::VecDeque}, - sync::{Arc, Weak}, -}; +use alloc::collections::vec_deque::VecDeque; +use alloc::sync::Arc; use core::sync::atomic::{AtomicU32, Ordering}; -use eonix_mm::address::VAddr; + use eonix_sync::{ - AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLockReadGuard, SpinGuard, - UnlockableGuard as _, UnlockedGuard as _, + AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLockReadGuard, + SpinGuard, UnlockableGuard as _, UnlockedGuard as _, +}; +use intrusive_collections::{ + intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink, }; use pointers::BorrowedArc; use posix_types::constants::{ - CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_PGID, P_PIDFD, + CLD_CONTINUED, CLD_DUMPED, CLD_EXITED, CLD_KILLED, CLD_STOPPED, P_ALL, + P_PGID, P_PID, P_PIDFD, }; -use posix_types::constants::{P_ALL, P_PID}; use posix_types::signal::Signal; use posix_types::SIGNAL_COREDUMP; +use super::signal::RaiseResult; +use super::thread::{ProcessThreads, ThreadBuilder}; +use super::{ProcessGroup, ProcessList, Session, Thread}; +use crate::kernel::constants::{ECHILD, EINTR, EINVAL, EPERM, ESRCH}; +use crate::kernel::mem::MMList; +use crate::kernel::task::{CloneArgs, CloneFlags}; +use crate::prelude::*; +use crate::rcu::{call_rcu, RCUPointer, RCUReadGuard}; +use crate::sync::CondVar; + pub struct ProcessBuilder { mm_list: Option<MMList>, exit_signal: Option<Signal>, @@ -39,7 +37,6 @@ pub struct ProcessBuilder { pid: Option<u32>, } -#[derive(Debug)] pub struct Process { /// Process id /// @@ -51,8 +48,6 @@ pub struct Process { pub exit_signal: Option<Signal>, - pub shm_areas: Spin<BTreeMap<VAddr, usize>>, - /// Parent process /// /// `parent` must be valid during the whole life of the process. @@ -72,14 +67,55 @@ pub struct Process { /// The only case where it may be `None` is when the process is kernel thread. pub(super) session: RCUPointer<Session>, - /// All things related to the process list. - pub(super) inner: Locked<ProcessInner, ProcessList>, + pub children: Locked<RBTree<ProcessChildren>, ProcessList>, + pub threads: Locked<RBTree<ProcessThreads>, ProcessList>, + + all_procs_link: RBTreeAtomicLink, + group_procs_link: RBTreeAtomicLink, + siblings_link: RBTreeAtomicLink, } -#[derive(Debug)] -pub(super) struct ProcessInner { - pub(super) children: BTreeMap<u32, Weak<Process>>, - pub(super) threads: BTreeMap<u32, Weak<Thread>>, +intrusive_adapter!(pub AllProcs = Arc<Process>: Process { + all_procs_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub GroupProcs = Arc<Process>: Process { + group_procs_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub ProcessChildren = Arc<Process>: Process { + siblings_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllProcs { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pid + } +} + +impl KeyAdapter<'_> for GroupProcs { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pid + } +} + +impl KeyAdapter<'_> for ProcessChildren { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pid + } } #[derive(Debug)] @@ -154,7 +190,9 @@ impl WaitType { pub fn to_wstatus(self) -> u32 { match self { WaitType::Exited(status) => (status & 0xff) << 8, - WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => signal.into_raw() | 0x80, + WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => { + signal.into_raw() | 0x80 + } WaitType::Signaled(signal) => signal.into_raw(), WaitType::Stopped(signal) => 0x7f | (signal.into_raw() << 8), WaitType::Continued => 0xffff, @@ -165,7 +203,9 @@ impl WaitType { // TODO: CLD_TRAPPED match self { WaitType::Exited(status) => (status, CLD_EXITED), - WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => (signal.into_raw(), CLD_DUMPED), + WaitType::Signaled(signal @ SIGNAL_COREDUMP!()) => { + (signal.into_raw(), CLD_DUMPED) + } WaitType::Signaled(signal) => (signal.into_raw(), CLD_KILLED), WaitType::Stopped(signal) => (signal.into_raw(), CLD_STOPPED), WaitType::Continued => (Signal::SIGCONT.into_raw(), CLD_CONTINUED), @@ -200,7 +240,9 @@ impl ProcessBuilder { } } - pub async fn clone_from(mut self, process: Arc<Process>, clone_args: &CloneArgs) -> Self { + pub async fn clone_from( + mut self, process: Arc<Process>, clone_args: &CloneArgs, + ) -> Self { let mm_list = if clone_args.flags.contains(CloneFlags::CLONE_VM) { process.mm_list.new_shared().await } else { @@ -249,30 +291,36 @@ impl ProcessBuilder { self } - pub fn build(self, process_list: &mut ProcessList) -> (Arc<Thread>, Arc<Process>) { + pub fn build( + self, process_list: &mut ProcessList, + ) -> (Arc<Thread>, Arc<Process>) { let mm_list = self.mm_list.unwrap_or_else(|| MMList::new()); let process = Arc::new(Process { pid: self.pid.expect("should set pid before building"), wait_list: WaitList::new(), mm_list, - shm_areas: Spin::new(BTreeMap::new()), exit_signal: self.exit_signal, parent: RCUPointer::empty(), pgroup: RCUPointer::empty(), session: RCUPointer::empty(), - inner: Locked::new( - ProcessInner { - children: BTreeMap::new(), - threads: BTreeMap::new(), - }, + children: Locked::new( + RBTree::new(ProcessChildren::NEW), process_list, ), + threads: Locked::new( + RBTree::new(ProcessThreads::NEW), + process_list, + ), + all_procs_link: RBTreeAtomicLink::new(), + group_procs_link: RBTreeAtomicLink::new(), + siblings_link: RBTreeAtomicLink::new(), }); process_list.add_process(&process); - let thread_builder = self.thread_builder.expect("Thread builder is not set"); + let thread_builder = + self.thread_builder.expect("Thread builder is not set"); let thread = thread_builder .process(process.clone()) .tid(process.pid) @@ -288,10 +336,7 @@ impl ProcessBuilder { pgroup.add_member(&process, process_list.prove_mut()); pgroup } - None => ProcessGroupBuilder::new() - .leader(&process) - .session(session.clone()) - .build(process_list), + None => ProcessGroup::new(&process, &session, process_list), }; if let Some(parent) = &self.parent { @@ -311,37 +356,41 @@ impl ProcessBuilder { impl Process { pub fn raise(&self, signal: Signal, procs: Proof<'_, ProcessList>) { - let inner = self.inner.access(procs); - for thread in inner.threads.values().map(|t| t.upgrade().unwrap()) { + let threads = self.threads.access(procs); + for thread in threads.iter() { if let RaiseResult::Finished = thread.raise(signal) { break; } } } - pub(super) fn add_child(&self, child: &Arc<Process>, procs: ProofMut<'_, ProcessList>) { - assert!(self - .inner - .access_mut(procs) - .children - .insert(child.pid, Arc::downgrade(child)) - .is_none()); + pub fn add_child( + &self, child: &Arc<Process>, procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_procs_link.is_linked(), "Dead process"); + self.children.access_mut(procs).insert(child.clone()); + } + + pub fn add_thread( + &self, thread: &Arc<Thread>, procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_procs_link.is_linked(), "Dead process"); + self.threads.access_mut(procs).insert(thread.clone()); } - pub(super) fn add_thread(&self, thread: &Arc<Thread>, procs: ProofMut<'_, ProcessList>) { - assert!(self - .inner - .access_mut(procs) - .threads - .insert(thread.tid, Arc::downgrade(thread)) - .is_none()); + pub fn remove_thread( + &self, thread: &Arc<Thread>, procs: ProofMut<'_, ProcessList>, + ) { + let threads = self.threads.access_mut(procs); + + assert!( + threads.find_mut(&thread.tid).remove().is_some(), + "Not a member" + ); } pub async fn wait( - &self, - wait_id: WaitId, - no_block: bool, - trace_stop: bool, + &self, wait_id: WaitId, no_block: bool, trace_stop: bool, trace_continue: bool, ) -> KResult<Option<WaitObject>> { let wait_object = { @@ -361,12 +410,7 @@ impl Process { break object; } - if self - .inner - .access(waits.process_list.prove()) - .children - .is_empty() - { + if self.children.access(waits.process_list.prove()).is_empty() { return Err(ECHILD); } @@ -382,12 +426,12 @@ impl Process { Ok(Some(wait_object)) } else { let mut procs = ProcessList::get().write().await; - procs.remove_process(wait_object.pid).await; + procs.remove_process(wait_object.pid); assert!(self - .inner - .access_mut(procs.prove_mut()) .children - .remove(&wait_object.pid) + .access_mut(procs.prove_mut()) + .find_mut(&wait_object.pid) + .remove() .is_some()); Ok(Some(wait_object)) @@ -403,15 +447,17 @@ impl Process { if process_list.try_find_session(self.pid).is_some() { return Err(EPERM); } + + self.pgroup(process_list.prove()) + .remove_member(self, &mut process_list); + let session = Session::new(self, &mut process_list); - let pgroup = ProcessGroupBuilder::new() - .leader(self) - .session(session.clone()) - .build(&mut process_list); + let pgroup = ProcessGroup::new(self, &session, &mut process_list); - let old_session = unsafe { self.session.swap(Some(session.clone())) }.unwrap(); - let old_pgroup = unsafe { self.pgroup.swap(Some(pgroup.clone())) }.unwrap(); - old_pgroup.remove_member(self.pid, process_list.prove_mut()); + let old_session = + unsafe { self.session.swap(Some(session.clone())) }.unwrap(); + let old_pgroup = + unsafe { self.pgroup.swap(Some(pgroup.clone())) }.unwrap(); call_rcu(move || { drop(old_session); @@ -424,47 +470,54 @@ impl Process { /// Set the process group id of the process to `pgid`. /// /// This function does the actual work. - fn do_setpgid(self: &Arc<Self>, pgid: u32, procs: &mut ProcessList) -> KResult<()> { + fn do_setpgid( + self: &Arc<Self>, pgid: u32, procs: &mut ProcessList, + ) -> KResult<()> { // SAFETY: We are holding the process list lock. let session = unsafe { self.session.load_locked().unwrap() }; - let pgroup = unsafe { self.pgroup.load_locked().unwrap() }; // Changing the process group of a session leader is not allowed. if session.sid == self.pid { return Err(EPERM); } - let new_pgroup = if let Some(new_pgroup) = procs.try_find_pgroup(pgid) { + let cur_pgroup = self.pgroup(procs.prove()).clone(); + let existing_pgroup = procs.try_find_pgroup(pgid); + + if let Some(new_pgroup) = &existing_pgroup { // Move us to an existing process group. // Check that the two groups are in the same session. - if new_pgroup.session.upgrade().unwrap().sid != session.sid { + if new_pgroup.session.sid != session.sid { return Err(EPERM); } // If we are already in the process group, we are done. - if new_pgroup.pgid == pgroup.pgid { + if new_pgroup.pgid == cur_pgroup.pgid { return Ok(()); } - - new_pgroup.add_member(self, procs.prove_mut()); - - new_pgroup } else { // Create a new process group only if `pgid` matches our `pid`. if pgid != self.pid { return Err(EPERM); } + } - ProcessGroupBuilder::new() - .leader(self) - .session(session.clone()) - .build(procs) - }; + // Permission checks done. Let's do the actual work. + cur_pgroup.remove_member(self, procs); - pgroup.remove_member(self.pid, procs.prove_mut()); + let new_pgroup; + if let Some(pgroup) = existing_pgroup { + pgroup.add_member(self, procs.prove_mut()); + new_pgroup = pgroup; + } else { + new_pgroup = ProcessGroup::new(self, &session, procs); + } - let old_pgroup = unsafe { self.pgroup.swap(Some(new_pgroup)) }.unwrap(); - call_rcu(move || drop(old_pgroup)); + unsafe { + // SAFETY: `cur_pgroup` held above. + self.pgroup.swap(Some(new_pgroup)); + } + call_rcu(move || drop(cur_pgroup)); Ok(()) } @@ -482,15 +535,14 @@ impl Process { let child = { // If `pid` refers to one of our children, the thread leaders must be // in out children list. - let children = &self.inner.access(procs.prove()).children; - let child = { - let child = children.get(&pid); - child.and_then(Weak::upgrade).ok_or(ESRCH)? - }; + let children = self.children.access(procs.prove()); + let child = children.find(&pid).clone_pointer().ok_or(ESRCH)?; // Changing the process group of a child is only allowed // if we are in the same session. - if child.session(procs.prove()).sid != self.session(procs.prove()).sid { + if child.session(procs.prove()).sid + != self.session(procs.prove()).sid + { return Err(EPERM); } @@ -504,39 +556,52 @@ impl Process { } /// Provide locked (consistent) access to the session. - pub fn session<'r>(&'r self, _procs: Proof<'r, ProcessList>) -> BorrowedArc<'r, Session> { + pub fn session<'r>( + &'r self, _procs: Proof<'r, ProcessList>, + ) -> BorrowedArc<'r, Session> { // SAFETY: We are holding the process list lock. unsafe { self.session.load_locked() }.unwrap() } /// Provide locked (consistent) access to the process group. - pub fn pgroup<'r>(&'r self, _procs: Proof<'r, ProcessList>) -> BorrowedArc<'r, ProcessGroup> { + pub fn pgroup<'r>( + &'r self, _procs: Proof<'r, ProcessList>, + ) -> BorrowedArc<'r, ProcessGroup> { // SAFETY: We are holding the process list lock. unsafe { self.pgroup.load_locked() }.unwrap() } /// Provide locked (consistent) access to the parent process. - pub fn parent<'r>(&'r self, _procs: Proof<'r, ProcessList>) -> BorrowedArc<'r, Process> { + pub fn parent<'r>( + &'r self, _procs: Proof<'r, ProcessList>, + ) -> BorrowedArc<'r, Process> { // SAFETY: We are holding the process list lock. unsafe { self.parent.load_locked() }.unwrap() } /// Provide RCU locked (maybe inconsistent) access to the session. - pub fn session_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<Session>> { + pub fn session_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<'_, Session>> { self.session.load().unwrap() } /// Provide RCU locked (maybe inconsistent) access to the process group. - pub fn pgroup_rcu(&self) -> RCUReadGuard<'_, BorrowedArc<ProcessGroup>> { + pub fn pgroup_rcu( + &self, + ) -> RCUReadGuard<'_, BorrowedArc<'_, ProcessGroup>> { self.pgroup.load().unwrap() } /// Provide RCU locked (maybe inconsistent) access to the parent process. - pub fn parent_rcu(&self) -> Option<RCUReadGuard<'_, BorrowedArc<Process>>> { + pub fn parent_rcu( + &self, + ) -> Option<RCUReadGuard<'_, BorrowedArc<'_, Process>>> { self.parent.load() } - pub fn notify(&self, signal: Option<Signal>, wait: WaitObject, procs: Proof<'_, ProcessList>) { + pub fn notify( + &self, signal: Option<Signal>, wait: WaitObject, + procs: Proof<'_, ProcessList>, + ) { self.wait_list.notify(wait); if let Some(signal) = signal { @@ -569,7 +634,7 @@ impl WaitList { self.cv_wait_procs.notify_all(); } - pub fn drain_exited(&self) -> DrainExited { + pub fn drain_exited(&self) -> DrainExited<'_> { DrainExited { wait_procs: self.wait_procs.lock(), } @@ -578,7 +643,9 @@ impl WaitList { /// # Safety /// Locks `ProcessList` and `WaitList` at the same time. When `wait` is called, /// releases the lock on `ProcessList` and `WaitList` and waits on `cv_wait_procs`. - pub async fn entry(&self, wait_id: WaitId, want_stop: bool, want_continue: bool) -> Entry { + pub async fn entry( + &self, wait_id: WaitId, want_stop: bool, want_continue: bool, + ) -> Entry<'_, '_, '_> { Entry { process_list: ProcessList::get().read().await, wait_procs: self.wait_procs.lock(), @@ -609,8 +676,11 @@ impl Entry<'_, '_, '_> { WaitId::Any => true, WaitId::Pid(pid) => item.pid == pid, WaitId::Pgid(pgid) => { - if let Some(process) = self.process_list.try_find_process(item.pid) { - return process.pgroup(self.process_list.prove()).pgid == pgid; + if let Some(process) = + self.process_list.try_find_process(item.pid) + { + return process.pgroup(self.process_list.prove()).pgid + == pgid; } false } @@ -624,7 +694,9 @@ impl Entry<'_, '_, '_> { } } - pub fn wait(self, no_block: bool) -> impl core::future::Future<Output = KResult<Self>> + Send { + pub fn wait( + self, no_block: bool, + ) -> impl core::future::Future<Output = KResult<Self>> + Send { let wait_procs = self.wait_procs.unlock(); async move { diff --git a/src/kernel/task/process_group.rs b/src/kernel/task/process_group.rs index 137c5191..8c708b5c 100644 --- a/src/kernel/task/process_group.rs +++ b/src/kernel/task/process_group.rs @@ -1,87 +1,121 @@ -use super::{Process, ProcessList, Session}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, +use alloc::sync::{Arc, Weak}; + +use eonix_sync::{AsProofMut, Locked, Proof, ProofMut}; +use intrusive_collections::{ + intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink, }; -use eonix_sync::{Locked, Proof, ProofMut}; use posix_types::signal::Signal; -pub struct ProcessGroupBuilder { - pgid: Option<u32>, - leader: Option<Weak<Process>>, - session: Option<Arc<Session>>, -} +use super::process::GroupProcs; +use super::{Process, ProcessList, Session}; -#[derive(Debug)] pub struct ProcessGroup { pub pgid: u32, - pub _leader: Weak<Process>, - pub session: Weak<Session>, + pub leader: Weak<Process>, + pub session: Arc<Session>, - pub processes: Locked<BTreeMap<u32, Weak<Process>>, ProcessList>, + pub procs: Locked<RBTree<GroupProcs>, ProcessList>, + + all_groups_link: RBTreeAtomicLink, + session_groups_link: RBTreeAtomicLink, } -impl ProcessGroupBuilder { - pub const fn new() -> Self { - Self { - pgid: None, - leader: None, - session: None, - } - } +intrusive_adapter!(pub AllGroups = Arc<ProcessGroup>: ProcessGroup { + all_groups_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub SessionGroups = Arc<ProcessGroup>: ProcessGroup { + session_groups_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllGroups { + type Key = u32; - pub fn leader(mut self, leader: &Arc<Process>) -> Self { - self.pgid = Some(leader.pid); - self.leader = Some(Arc::downgrade(leader)); - self + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pgid } +} + +impl KeyAdapter<'_> for SessionGroups { + type Key = u32; - pub fn session(mut self, session: Arc<Session>) -> Self { - self.session = Some(session); - self + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.pgid } +} - pub fn build(self, process_list: &mut ProcessList) -> Arc<ProcessGroup> { - let pgid = self.pgid.expect("PGID is not set"); - let leader = self.leader.expect("Leader is not set"); - let session = self.session.expect("Session is not set"); +impl ProcessGroup { + /// Create a pgroup and add it to the global pgroup list. + /// Add the pgroup to the session. + /// + /// # Panics + /// Panics if `leader` is already in some pgroup. + pub fn new( + leader: &Arc<Process>, + session: &Arc<Session>, + procs: &mut ProcessList, + ) -> Arc<Self> { + let pgid = leader.pid; + let pgroup_procs = { + let mut list = RBTree::new(GroupProcs::new()); + list.insert(leader.clone()); + list + }; let pgroup = Arc::new(ProcessGroup { pgid, - session: Arc::downgrade(&session), - processes: Locked::new(BTreeMap::from([(pgid, leader.clone())]), process_list), - _leader: leader, + session: session.clone(), + procs: Locked::new(pgroup_procs, procs), + leader: Arc::downgrade(leader), + all_groups_link: RBTreeAtomicLink::new(), + session_groups_link: RBTreeAtomicLink::new(), }); - process_list.add_pgroup(&pgroup); - session.add_member(process_list, &pgroup); + procs.add_pgroup(&pgroup); + session.add_member(&pgroup, procs.prove_mut()); pgroup } -} -impl ProcessGroup { - pub(super) fn add_member(&self, process: &Arc<Process>, procs: ProofMut<'_, ProcessList>) { - assert!(self - .processes - .access_mut(procs) - .insert(process.pid, Arc::downgrade(process)) - .is_none()); + /// Add `process` to the pgroup. + /// + /// # Panics + /// Panics if `process` is already in some pgroup or the pgroup is dead. + pub fn add_member( + &self, + process: &Arc<Process>, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_groups_link.is_linked(), "Dead pgroup"); + self.procs.access_mut(procs).insert(process.clone()); } - pub(super) fn remove_member(&self, pid: u32, procs: ProofMut<'_, ProcessList>) { - let processes = self.processes.access_mut(procs); - assert!(processes.remove(&pid).is_some()); - if processes.is_empty() { - self.session - .upgrade() - .unwrap() - .remove_member(self.pgid, procs); + pub fn remove_member( + self: &Arc<Self>, + process: &Arc<Process>, + procs: &mut ProcessList, + ) { + let members = self.procs.access_mut(procs.prove_mut()); + assert!( + members.find_mut(&process.pid).remove().is_some(), + "Not a member" + ); + + if !members.is_empty() { + return; } + + self.session.remove_member(self, procs); + procs.remove_pgroup(self); } pub fn raise(&self, signal: Signal, procs: Proof<'_, ProcessList>) { - let processes = self.processes.access(procs); - for process in processes.values().map(|p| p.upgrade().unwrap()) { + let members = self.procs.access(procs); + for process in members.iter() { process.raise(signal, procs); } } diff --git a/src/kernel/task/process_list.rs b/src/kernel/task/process_list.rs index af073e84..ccce8e2c 100644 --- a/src/kernel/task/process_list.rs +++ b/src/kernel/task/process_list.rs @@ -1,36 +1,41 @@ -use core::sync::atomic::Ordering; - -use super::{Process, ProcessGroup, Session, Thread, WaitObject, WaitType}; -use crate::{ - kernel::{task::futex_wake, user::UserPointerMut}, - rcu::rcu_sync, -}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, +use alloc::collections::vec_deque::VecDeque; +use alloc::sync::Arc; +use core::pin::pin; + +use eonix_runtime::scheduler::RUNTIME; +use eonix_sync::{AsProof as _, AsProofMut as _, RwLock, Spin, WaitList}; +use intrusive_collections::RBTree; + +use super::loader::LoadInfo; +use super::process::AllProcs; +use super::process_group::AllGroups; +use super::session::AllSessions; +use super::thread::AllThreads; +use super::{ + alloc_pid, Process, ProcessBuilder, ProcessGroup, Session, Thread, + ThreadBuilder, WaitObject, }; -use eonix_mm::address::Addr; -use eonix_sync::{AsProof as _, AsProofMut as _, RwLock}; +use crate::rcu::call_rcu; pub struct ProcessList { /// The init process. init: Option<Arc<Process>>, /// All threads. - threads: BTreeMap<u32, Arc<Thread>>, + threads: RBTree<AllThreads>, /// All processes. - processes: BTreeMap<u32, Weak<Process>>, + procs: RBTree<AllProcs>, /// All process groups. - pgroups: BTreeMap<u32, Weak<ProcessGroup>>, + pgroups: RBTree<AllGroups>, /// All sessions. - sessions: BTreeMap<u32, Weak<Session>>, + sessions: RBTree<AllSessions>, } static GLOBAL_PROC_LIST: RwLock<ProcessList> = RwLock::new(ProcessList { init: None, - threads: BTreeMap::new(), - processes: BTreeMap::new(), - pgroups: BTreeMap::new(), - sessions: BTreeMap::new(), + threads: RBTree::new(AllThreads::NEW), + procs: RBTree::new(AllProcs::NEW), + pgroups: RBTree::new(AllGroups::NEW), + sessions: RBTree::new(AllSessions::NEW), }); impl ProcessList { @@ -39,46 +44,69 @@ impl ProcessList { } pub fn add_session(&mut self, session: &Arc<Session>) { - self.sessions.insert(session.sid, Arc::downgrade(session)); + self.sessions.insert(session.clone()); } pub fn add_pgroup(&mut self, pgroup: &Arc<ProcessGroup>) { - self.pgroups.insert(pgroup.pgid, Arc::downgrade(pgroup)); + self.pgroups.insert(pgroup.clone()); } pub fn add_process(&mut self, process: &Arc<Process>) { - self.processes.insert(process.pid, Arc::downgrade(process)); + self.procs.insert(process.clone()); } pub fn add_thread(&mut self, thread: &Arc<Thread>) { - self.threads.insert(thread.tid, thread.clone()); + self.threads.insert(thread.clone()); } - pub async fn remove_process(&mut self, pid: u32) { + pub fn remove_process(&mut self, pid: u32) { + let Some(proc) = self.procs.find_mut(&pid).remove() else { + panic!("Process {} not found", pid); + }; + // Thread group leader has the same tid as the pid. - if let Some(thread) = self.threads.remove(&pid) { - self.processes.remove(&pid); - - // SAFETY: We wait until all references are dropped below with `rcu_sync()`. - let session = unsafe { thread.process.session.swap(None) }.unwrap(); - let pgroup = unsafe { thread.process.pgroup.swap(None) }.unwrap(); - let _parent = unsafe { thread.process.parent.swap(None) }.unwrap(); - pgroup.remove_member(pid, self.prove_mut()); - rcu_sync().await; - - if Arc::strong_count(&pgroup) == 1 { - self.pgroups.remove(&pgroup.pgid); - } + let Some(thd) = self.threads.find_mut(&pid).remove() else { + panic!("Thread {} not found", pid); + }; - if Arc::strong_count(&session) == 1 { - self.sessions.remove(&session.sid); - } - } else { - panic!("Process {} not found", pid); - } + proc.remove_thread(&thd, self.prove_mut()); + + // SAFETY: `call_rcu` below. + let session = unsafe { proc.session.swap(None) }.unwrap(); + let pgroup = unsafe { proc.pgroup.swap(None) }.unwrap(); + let parent = unsafe { proc.parent.swap(None) }.unwrap(); + + pgroup.remove_member(&proc, self); + + call_rcu(move || { + drop(session); + drop(pgroup); + drop(parent); + }); } - pub fn set_init_process(&mut self, init: Arc<Process>) { + pub fn remove_thread(&mut self, thread: &Arc<Thread>) { + assert!( + self.threads.find_mut(&thread.tid).remove().is_some(), + "Double remove" + ); + } + + pub fn remove_session(&mut self, session: &Arc<Session>) { + assert!( + self.sessions.find_mut(&session.sid).remove().is_some(), + "Double remove" + ); + } + + pub fn remove_pgroup(&mut self, pgroup: &Arc<ProcessGroup>) { + assert!( + self.pgroups.find_mut(&pgroup.pgid).remove().is_some(), + "Double remove" + ); + } + + fn set_init_process(&mut self, init: Arc<Process>) { let old_init = self.init.replace(init); assert!(old_init.is_none(), "Init process already set"); } @@ -87,61 +115,80 @@ impl ProcessList { self.init.as_ref().unwrap() } - pub fn try_find_thread(&self, tid: u32) -> Option<&Arc<Thread>> { - self.threads.get(&tid) + pub fn try_find_thread(&self, tid: u32) -> Option<Arc<Thread>> { + self.threads.find(&tid).clone_pointer() } pub fn try_find_process(&self, pid: u32) -> Option<Arc<Process>> { - self.processes.get(&pid).and_then(Weak::upgrade) + self.procs.find(&pid).clone_pointer() } pub fn try_find_pgroup(&self, pgid: u32) -> Option<Arc<ProcessGroup>> { - self.pgroups.get(&pgid).and_then(Weak::upgrade) + self.pgroups.find(&pgid).clone_pointer() } pub fn try_find_session(&self, sid: u32) -> Option<Arc<Session>> { - self.sessions.get(&sid).and_then(Weak::upgrade) - } - - /// Make the process a zombie and notify the parent. - /// # Safety - /// This function will destroy the process and all its threads. - /// It is the caller's responsibility to ensure that the process is not - /// running or will not run after this function is called. - pub async unsafe fn do_exit( - &mut self, - thread: &Thread, - exit_status: WaitType, - is_exiting_group: bool, - ) { - let process = thread.process.clone(); - - if process.pid == 1 { - panic!("init exited"); - } + self.sessions.find(&sid).clone_pointer() + } - let inner = process.inner.access_mut(self.prove_mut()); + pub async fn sys_init(load_info: LoadInfo) { + let thread_builder = ThreadBuilder::new() + .name(Arc::from(&b"busybox"[..])) + .entry(load_info.entry_ip, load_info.sp); - thread.dead.store(true, Ordering::SeqCst); + let mut process_list = ProcessList::get().write().await; + let (thread, process) = ProcessBuilder::new() + .pid(alloc_pid()) + .mm_list(load_info.mm_list) + .thread_builder(thread_builder) + .build(&mut process_list); - if is_exiting_group { - // TODO: Send SIGKILL to all threads. - // todo!() - } + process_list.set_init_process(process); - if thread.tid != process.pid { - self.threads.remove(&thread.tid); - inner.threads.remove(&thread.tid).unwrap(); - } + RUNTIME.spawn(Reaper::daemon()); + RUNTIME.spawn(thread.run()); + } + + pub fn send_to_reaper(thread: Arc<Thread>) { + GLOBAL_REAPER.reap_list.lock().push_back(thread); + GLOBAL_REAPER.wait.notify_one(); + } +} - if let Some(clear_ctid) = thread.get_clear_ctid() { - let _ = UserPointerMut::new(clear_ctid).unwrap().write(0u32); +struct Reaper { + reap_list: Spin<VecDeque<Arc<Thread>>>, + wait: WaitList, +} - let _ = futex_wake(clear_ctid.addr(), None, 1).await; +static GLOBAL_REAPER: Reaper = Reaper { + reap_list: Spin::new(VecDeque::new()), + wait: WaitList::new(), +}; + +impl Reaper { + async fn reap(&self, thread: Arc<Thread>) { + let exit_status = thread + .exit_status + .lock() + .take() + .expect("Exited thread with no exit status"); + + let process = &thread.process; + + if process.pid == 1 && thread.tid == process.pid { + panic!("init exited: {}", alloc_pid()); } - if let Some(robust_list) = thread.get_robust_list() { - let _ = robust_list.wake_all().await; + let mut procs = ProcessList::get().write().await; + + if thread.tid != process.pid { + let threads = process.threads.access_mut(procs.prove_mut()); + assert!( + threads.find_mut(&thread.tid).remove().is_some(), + "Thread gone?" + ); + + procs.remove_thread(&thread); } // main thread exit @@ -150,49 +197,61 @@ impl ProcessList { thread.files.close_all().await; + let session = process.session(procs.prove()).clone(); // If we are the session leader, we should drop the control terminal. - if process.session(self.prove()).sid == process.pid { - if let Some(terminal) = process.session(self.prove()).drop_control_terminal().await - { - terminal.drop_session().await; + if session.sid == process.pid { + if let Some(terminal) = session.control_terminal() { + terminal.drop_session(procs.prove()); } } // Release the MMList as well as the page table. - unsafe { - // SAFETY: We are exiting the process, so no one might be using it. - process.mm_list.replace(None); - } + process.mm_list.release(); // Make children orphans (adopted by init) - { - let init = self.init_process(); - inner.children.retain(|_, child| { - let child = child.upgrade().unwrap(); - // SAFETY: `child.parent` must be ourself. So we don't need to free it. - unsafe { child.parent.swap(Some(init.clone())) }; - init.add_child(&child, self.prove_mut()); - - false - }); + let init = procs.init_process(); + let children = process.children.access_mut(procs.prove_mut()); + for child in children.take() { + // XXX: May buggy. Check here again. + // SAFETY: `child.parent` must be ourself. + // So we don't need to free it. + unsafe { child.parent.swap(Some(init.clone())) }; + init.add_child(&child, procs.prove_mut()); } - let mut init_notify = self.init_process().notify_batch(); + let mut init_notify = procs.init_process().notify_batch(); process .wait_list .drain_exited() .into_iter() .for_each(|item| init_notify.notify(item)); - init_notify.finish(self.prove()); + init_notify.finish(procs.prove()); - process.parent(self.prove()).notify( + process.parent(procs.prove()).notify( process.exit_signal, WaitObject { pid: process.pid, code: exit_status, }, - self.prove(), + procs.prove(), ); } } + + async fn daemon() { + let me = &GLOBAL_REAPER; + + loop { + let mut wait = pin!(me.wait.prepare_to_wait()); + wait.as_mut().add_to_wait_list(); + + let thd_to_reap = me.reap_list.lock().pop_front(); + if let Some(thd_to_reap) = thd_to_reap { + me.reap(thd_to_reap).await; + continue; + } + + wait.await; + } + } } diff --git a/src/kernel/task/session.rs b/src/kernel/task/session.rs index a7b57afd..899aa395 100644 --- a/src/kernel/task/session.rs +++ b/src/kernel/task/session.rs @@ -1,117 +1,170 @@ -use super::{Process, ProcessGroup, ProcessList, Thread}; -use crate::kernel::constants::EPERM; -use crate::{kernel::Terminal, prelude::*}; -use alloc::{ - collections::btree_map::BTreeMap, - sync::{Arc, Weak}, +use alloc::sync::{Arc, Weak}; + +use eonix_sync::{AsProof as _, AsProofMut, Locked, Proof, ProofMut}; +use intrusive_collections::{ + intrusive_adapter, KeyAdapter, RBTree, RBTreeAtomicLink, }; -use eonix_sync::{AsProof as _, AsProofMut as _, Locked, Proof, ProofMut, RwLock}; use posix_types::signal::Signal; -#[derive(Debug)] +use super::process_group::SessionGroups; +use super::{Process, ProcessGroup, ProcessList}; +use crate::kernel::constants::EPERM; +use crate::kernel::Terminal; +use crate::prelude::*; + struct SessionJobControl { - /// Foreground process group - foreground: Weak<ProcessGroup>, + foreground: Option<Arc<ProcessGroup>>, control_terminal: Option<Arc<Terminal>>, } -#[allow(dead_code)] -#[derive(Debug)] pub struct Session { pub sid: u32, pub leader: Weak<Process>, - job_control: RwLock<SessionJobControl>, + job_control: Spin<SessionJobControl>, + groups: Locked<RBTree<SessionGroups>, ProcessList>, + all_sessions_link: RBTreeAtomicLink, +} - groups: Locked<BTreeMap<u32, Weak<ProcessGroup>>, ProcessList>, +intrusive_adapter!(pub AllSessions = Arc<Session>: Session { + all_sessions_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllSessions { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.sid + } } impl Session { /// Create a session and add it to the global session list. - pub fn new(leader: &Arc<Process>, process_list: &mut ProcessList) -> Arc<Self> { + pub fn new(leader: &Arc<Process>, proclist: &mut ProcessList) -> Arc<Self> { let session = Arc::new(Self { sid: leader.pid, leader: Arc::downgrade(leader), - job_control: RwLock::new(SessionJobControl { - foreground: Weak::new(), + job_control: Spin::new(SessionJobControl { + foreground: None, control_terminal: None, }), - groups: Locked::new( - BTreeMap::new(), - // SAFETY: `procs` must be the global process list, which won't be moved. - process_list, - ), + groups: Locked::new(RBTree::new(SessionGroups::NEW), proclist), + all_sessions_link: RBTreeAtomicLink::new(), }); - process_list.add_session(&session); + proclist.add_session(&session); session } - pub(super) fn add_member(&self, procs: &mut ProcessList, pgroup: &Arc<ProcessGroup>) { - let groups = self.groups.access_mut(procs.prove_mut()); - let old = groups.insert(pgroup.pgid, Arc::downgrade(pgroup)); - assert!(old.is_none(), "Process group already exists"); + pub fn add_member( + &self, + pgroup: &Arc<ProcessGroup>, + procs: ProofMut<'_, ProcessList>, + ) { + assert!(self.all_sessions_link.is_linked(), "Dead session"); + self.groups.access_mut(procs).insert(pgroup.clone()); } - pub(super) fn remove_member(&self, pgid: u32, procs: ProofMut<'_, ProcessList>) { - assert!(self.groups.access_mut(procs).remove(&pgid).is_some()); + pub fn remove_member( + self: &Arc<Self>, + pgroup: &Arc<ProcessGroup>, + procs: &mut ProcessList, + ) { + let members = self.groups.access_mut(procs.prove_mut()); + assert!( + members.find_mut(&pgroup.pgid).remove().is_some(), + "Not a member" + ); + + if let Some(fg_pgroup) = self.foreground_pgroup() { + if fg_pgroup.pgid == pgroup.pgid { + let _ = self.set_foreground_pgroup(None); + } + } + + if !members.is_empty() { + return; + } + + // Recycle dead session. + procs.remove_session(self); } - pub async fn foreground(&self) -> Option<Arc<ProcessGroup>> { - self.job_control.read().await.foreground.upgrade() + pub fn leader(&self) -> Option<Arc<Process>> { + self.leader.upgrade() + } + + pub fn foreground_pgroup(&self) -> Option<Arc<ProcessGroup>> { + self.job_control.lock().foreground.clone() + } + + pub fn control_terminal(&self) -> Option<Arc<Terminal>> { + self.job_control.lock().control_terminal.clone() } /// Set the foreground process group identified by `pgid`. /// The process group must belong to the session. - pub async fn set_foreground_pgid( + pub fn set_foreground_pgroup( &self, - pgid: u32, - procs: Proof<'_, ProcessList>, + pgroup: Option<&Arc<ProcessGroup>>, ) -> KResult<()> { - if let Some(group) = self.groups.access(procs).get(&pgid) { - self.job_control.write().await.foreground = group.clone(); - Ok(()) - } else { - // TODO: Check if the process group refers to an existing process group. - // That's not a problem though, the operation will fail anyway. - Err(EPERM) + if let Some(pgroup) = pgroup { + if pgroup.session.sid != self.sid { + return Err(EPERM); + } } + + self.job_control.lock().foreground = pgroup.cloned(); + Ok(()) } - /// Only session leaders can set the control terminal. - /// Make sure we've checked that before calling this function. - pub async fn set_control_terminal( + /// Set our controlling terminal to `terminal`. Only meant to be called by + /// the session leader. The pgroup that the session leader is in becomes the + /// new foreground pgroup. + /// + /// # Panics + /// Panics if we have a controlling terminal already + /// or the session leader is gone. + pub fn _set_control_terminal( self: &Arc<Self>, terminal: &Arc<Terminal>, - forced: bool, procs: Proof<'_, ProcessList>, - ) -> KResult<()> { - let mut job_control = self.job_control.write().await; - if let Some(_) = job_control.control_terminal.as_ref() { - if let Some(session) = terminal.session().await.as_ref() { - if session.sid == self.sid { - return Ok(()); - } - } - return Err(EPERM); - } - terminal.set_session(self, forced).await?; + ) { + let mut job_control = self.job_control.lock(); + let leader = self.leader().expect("Leader is gone?"); + + assert!( + job_control.control_terminal.is_none(), + "We have a controlling terminal already" + ); + job_control.control_terminal = Some(terminal.clone()); - job_control.foreground = Arc::downgrade(&Thread::current().process.pgroup(procs)); - Ok(()) + job_control.foreground = Some(leader.pgroup(procs).clone()); } /// Drop the control terminal reference inside the session. - /// DO NOT TOUCH THE TERMINAL'S SESSION FIELD. - pub async fn drop_control_terminal(&self) -> Option<Arc<Terminal>> { - let mut inner = self.job_control.write().await; - inner.foreground = Weak::new(); - inner.control_terminal.take() + /// Send SIGHUP and then SIGCONT to our foreground pgroup. + pub fn _drop_control_terminal(&self, procs: Proof<'_, ProcessList>) { + let foreground = { + let mut inner = self.job_control.lock(); + inner.control_terminal = None; + inner.foreground.take() + }; + + if let Some(foreground) = foreground { + foreground.raise(Signal::SIGHUP, procs); + foreground.raise(Signal::SIGCHLD, procs); + } } pub async fn raise_foreground(&self, signal: Signal) { - if let Some(fg) = self.foreground().await { - let procs = ProcessList::get().read().await; - fg.raise(signal, procs.prove()); - } + let Some(fg) = self.foreground_pgroup() else { + return; + }; + + let procs = ProcessList::get().read().await; + fg.raise(signal, procs.prove()); } } diff --git a/src/kernel/task/signal.rs b/src/kernel/task/signal.rs index d9970cad..0a7b580d 100644 --- a/src/kernel/task/signal.rs +++ b/src/kernel/task/signal.rs @@ -1,11 +1,10 @@ mod signal_action; -use super::{ProcessList, Thread, WaitObject, WaitType}; -use crate::kernel::constants::{EFAULT, EINVAL}; -use crate::{kernel::user::UserPointer, prelude::*}; use alloc::collections::binary_heap::BinaryHeap; use alloc::sync::Arc; -use core::{cmp::Reverse, task::Waker}; +use core::cmp::Reverse; +use core::task::Waker; + use eonix_hal::fpu::FpuState; use eonix_hal::traits::trap::RawTrapContext; use eonix_hal::trap::TrapContext; @@ -14,9 +13,13 @@ use eonix_sync::AsProof as _; use intrusive_collections::UnsafeRef; use posix_types::signal::{SigSet, Signal}; use posix_types::{SIGNAL_IGNORE, SIGNAL_NOW, SIGNAL_STOP}; +pub use signal_action::SignalAction; use signal_action::SignalActionList; -pub use signal_action::SignalAction; +use super::{ProcessList, Thread, WaitObject, WaitType}; +use crate::kernel::constants::{EFAULT, EINVAL}; +use crate::kernel::user::UserPointer; +use crate::prelude::*; pub(self) const SAVED_DATA_SIZE: usize = size_of::<TrapContext>() + size_of::<FpuState>() + size_of::<SigSet>(); @@ -168,10 +171,7 @@ impl SignalList { pub async fn handle(&self, trap_ctx: &mut TrapContext, fpu_state: &mut FpuState) { loop { let signal = { - let signal = match self.inner.lock().pop() { - Some(signal) => signal, - None => return, - }; + let Some(signal) = self.inner.lock().pop() else { return }; let handler = self.inner.lock().actions.get(signal); if let SignalAction::SimpleHandler { mask, .. } = &handler { @@ -246,7 +246,7 @@ impl SignalList { } signal => { // Default to terminate the thread. - Thread::current().force_kill(signal).await; + Thread::current().force_kill(signal); return; } } diff --git a/src/kernel/task/signal/signal_action.rs b/src/kernel/task/signal/signal_action.rs index 708f9802..2ee6dec9 100644 --- a/src/kernel/task/signal/signal_action.rs +++ b/src/kernel/task/signal/signal_action.rs @@ -1,22 +1,35 @@ -use super::{KResult, SAVED_DATA_SIZE}; -use crate::{ - io::BufferFill as _, - kernel::{ - constants::{EFAULT, EINVAL}, - syscall::UserMut, - user::UserBuffer, - }, -}; -use alloc::{collections::btree_map::BTreeMap, sync::Arc}; +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::Arc; use core::arch::naked_asm; -use eonix_hal::{fpu::FpuState, traits::trap::RawTrapContext, trap::TrapContext}; + +use eonix_hal::fpu::FpuState; +use eonix_hal::traits::trap::RawTrapContext; +use eonix_hal::trap::TrapContext; use eonix_mm::address::{Addr as _, AddrOps as _, VAddr}; use eonix_sync::Spin; -use posix_types::{ - ctypes::Long, - signal::{SigAction, SigActionHandler, SigActionRestorer, SigSet, Signal, TryFromSigAction}, - SIGNAL_NOW, +use posix_types::ctypes::Long; +use posix_types::signal::{ + SigAction, SigActionHandler, SigActionRestorer, SigSet, Signal, + TryFromSigAction, }; +use posix_types::syscall_no::SYS_RT_SIGRETURN; +use posix_types::SIGNAL_NOW; + +use super::{KResult, SAVED_DATA_SIZE}; +use crate::io::BufferFill as _; +use crate::kernel::constants::{EFAULT, EINVAL}; +use crate::kernel::syscall::UserMut; +use crate::kernel::user::UserBuffer; + +macro_rules! vdso_sym_addr { + ($sym:expr) => {{ + const VDSO_START_VADDR: VAddr = VAddr::from(0x7f00_0000_0000); + let vdso_link_start = eonix_hal::extern_symbol_addr!(VDSO_START); + + eonix_hal::symbol_addr!($sym) - vdso_link_start + + VDSO_START_VADDR.addr() + }}; +} #[cfg(target_arch = "x86_64")] #[unsafe(naked)] @@ -45,21 +58,21 @@ unsafe extern "C" fn vdso_rt_sigreturn() { naked_asm!( "li a7, {sys_rt_sigreturn}", "ecall", - sys_rt_sigreturn = const posix_types::syscall_no::SYS_RT_SIGRETURN, + sys_rt_sigreturn = const SYS_RT_SIGRETURN, ); #[cfg(target_arch = "loongarch64")] naked_asm!( "li.d $a7, {sys_rt_sigreturn}", "syscall 0", - sys_rt_sigreturn = const posix_types::syscall_no::SYS_RT_SIGRETURN, + sys_rt_sigreturn = const SYS_RT_SIGRETURN, ); #[cfg(target_arch = "x86_64")] naked_asm!( "mov ${sys_rt_sigreturn}, %eax", "int $0x80", - sys_rt_sigreturn = const posix_types::syscall_no::SYS_RT_SIGRETURN, + sys_rt_sigreturn = const SYS_RT_SIGRETURN, options(att_syntax), ); } @@ -129,17 +142,16 @@ impl SignalActionList { impl SignalAction { /// # Might Sleep pub(super) fn handle( - self, - signal: Signal, - old_mask: SigSet, - trap_ctx: &mut TrapContext, + self, signal: Signal, old_mask: SigSet, trap_ctx: &mut TrapContext, fpu_state: &mut FpuState, ) -> KResult<()> { let SignalAction::SimpleHandler { handler, restorer, .. } = self else { - unreachable!("Default and Ignore actions should not be handled here"); + unreachable!( + "Default and Ignore actions should not be handled here" + ); }; let current_sp = VAddr::from(trap_ctx.get_stack_pointer()); @@ -167,31 +179,19 @@ impl SignalAction { target_arch = "riscv64", target_arch = "loongarch64" )))] - compile_error!("`vdso_sigreturn` is not implemented for this architecture"); + compile_error!( + "`vdso_sigreturn` is not implemented for this architecture" + ); #[cfg(target_arch = "x86_64")] { // TODO: Check and use `vdso_rt_sigreturn` for x86 as well. - static VDSO_SIGRETURN_ADDR: &'static unsafe extern "C" fn() = - &(vdso_rt_sigreturn as unsafe extern "C" fn()); - - unsafe { - // SAFETY: To prevent the compiler from optimizing this into `la` instructions - // and causing a linking error. - (VDSO_SIGRETURN_ADDR as *const _ as *const usize).read_volatile() - } + vdso_sym_addr!(vdso_rt_sigreturn) } #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] { - static VDSO_RT_SIGRETURN_ADDR: &'static unsafe extern "C" fn() = - &(vdso_rt_sigreturn as unsafe extern "C" fn()); - - unsafe { - // SAFETY: To prevent the compiler from optimizing this into `la` instructions - // and causing a linking error. - (VDSO_RT_SIGRETURN_ADDR as *const _ as *const usize).read_volatile() - } + vdso_sym_addr!(vdso_rt_sigreturn) } }; @@ -201,7 +201,8 @@ impl SignalAction { Some(return_address), &[Long::new_val(signal.into_raw() as _).get()], |vaddr, data| -> Result<(), u32> { - let mut buffer = UserBuffer::new(UserMut::new(vaddr), data.len())?; + let mut buffer = + UserBuffer::new(UserMut::new(vaddr), data.len())?; for ch in data.iter() { buffer.copy(&ch)?.ok_or(EFAULT)?; } diff --git a/src/kernel/task/thread.rs b/src/kernel/task/thread.rs index 11348e51..c30f4b4b 100644 --- a/src/kernel/task/thread.rs +++ b/src/kernel/task/thread.rs @@ -1,43 +1,38 @@ -use super::{ - signal::{RaiseResult, SignalList}, - stackful, Process, ProcessList, WaitType, -}; -use crate::{ - kernel::{ - interrupt::default_irq_handler, - syscall::{syscall_handlers, SyscallHandler, User, UserMut}, - task::{clone::CloneArgs, futex::RobustListHead, CloneFlags}, - timer::{should_reschedule, timer_interrupt}, - user::{UserPointer, UserPointerMut}, - vfs::{filearray::FileArray, FsContext}, - }, - prelude::*, -}; -use alloc::{alloc::Allocator, sync::Arc}; +use alloc::alloc::Allocator; +use alloc::sync::Arc; +use core::future::{poll_fn, Future}; +use core::pin::Pin; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicBool, Ordering}; +use core::task::{Context, Poll}; + use atomic_unique_refcell::AtomicUniqueRefCell; -use core::{ - future::{poll_fn, Future}, - pin::Pin, - ptr::NonNull, - sync::atomic::{AtomicBool, Ordering}, - task::{Context, Poll}, -}; -use eonix_hal::{ - fpu::FpuState, - processor::{UserTLS, CPU}, - traits::{ - fault::Fault, - fpu::RawFpuState as _, - trap::{RawTrapContext, TrapReturn, TrapType}, - }, - trap::TrapContext, -}; +use eonix_hal::fpu::FpuState; +use eonix_hal::traits::fault::Fault; +use eonix_hal::traits::fpu::RawFpuState as _; +use eonix_hal::traits::trap::{RawTrapContext, TrapReturn, TrapType}; +use eonix_hal::trap::TrapContext; +use eonix_log::println_warn; use eonix_mm::address::{Addr as _, VAddr}; use eonix_sync::AsProofMut as _; +use intrusive_collections::{intrusive_adapter, KeyAdapter, RBTreeAtomicLink}; use pointers::BorrowedArc; use posix_types::signal::Signal; use stalloc::UnsafeStalloc; +use super::signal::{RaiseResult, SignalList}; +use super::user_tls::UserTLS; +use super::{stackful, Process, ProcessList, WaitType}; +use crate::kernel::interrupt::default_irq_handler; +use crate::kernel::syscall::{syscall_handlers, SyscallHandler, User, UserMut}; +use crate::kernel::task::clone::CloneArgs; +use crate::kernel::task::{futex_exit, CloneFlags, RobustListHead}; +use crate::kernel::timer::{should_reschedule, timer_interrupt}; +use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::kernel::vfs::filearray::FileArray; +use crate::kernel::vfs::FsContext; +use crate::prelude::*; + #[eonix_percpu::define_percpu] static CURRENT_THREAD: Option<NonNull<Thread>> = None; @@ -89,10 +84,46 @@ pub struct Thread { pub fpu_state: AtomicUniqueRefCell<FpuState>, pub dead: AtomicBool, + pub exit_status: Spin<Option<WaitType>>, + + /// Link in the global thread list. + all_threads_link: RBTreeAtomicLink, + + /// Link in the process's thread list. + process_threads_link: RBTreeAtomicLink, inner: Spin<ThreadInner>, } +intrusive_adapter!(pub AllThreads = Arc<Thread>: Thread { + all_threads_link: RBTreeAtomicLink +}); +intrusive_adapter!(pub ProcessThreads = Arc<Thread>: Thread { + process_threads_link: RBTreeAtomicLink +}); + +impl KeyAdapter<'_> for AllThreads { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.tid + } +} + +impl KeyAdapter<'_> for ProcessThreads { + type Key = u32; + + fn get_key( + &self, + value: &'_ <Self::PointerOps as intrusive_collections::PointerOps>::Value, + ) -> Self::Key { + value.tid + } +} + impl ThreadBuilder { pub fn new() -> Self { Self { @@ -145,12 +176,16 @@ impl ThreadBuilder { self } - pub fn set_child_tid(mut self, set_child_tid: Option<UserMut<u32>>) -> Self { + pub fn set_child_tid( + mut self, set_child_tid: Option<UserMut<u32>>, + ) -> Self { self.set_child_tid = set_child_tid; self } - pub fn clear_child_tid(mut self, clear_child_tid: Option<UserMut<u32>>) -> Self { + pub fn clear_child_tid( + mut self, clear_child_tid: Option<UserMut<u32>>, + ) -> Self { self.clear_child_tid = clear_child_tid; self } @@ -166,20 +201,18 @@ impl ThreadBuilder { } pub fn entry(mut self, entry: VAddr, stack_pointer: VAddr) -> Self { - let mut trap_ctx = TrapContext::new(); - trap_ctx.set_user_mode(true); + let mut trap_ctx = TrapContext::new(true, true); trap_ctx.set_program_counter(entry.addr()); trap_ctx.set_stack_pointer(stack_pointer.addr()); - trap_ctx.set_interrupt_enabled(true); self.trap_ctx = Some(trap_ctx); self } /// Clone the thread from another thread. - pub fn clone_from(self, thread: &Thread, clone_args: &CloneArgs) -> KResult<Self> { - let inner = thread.inner.lock(); - + pub fn clone_from( + self, thread: &Thread, clone_args: &CloneArgs, + ) -> KResult<Self> { let mut trap_ctx = thread.trap_ctx.borrow().clone(); trap_ctx.set_user_return_value(0); @@ -205,18 +238,26 @@ impl ThreadBuilder { FileArray::new_cloned(&thread.files) }; - let signal_list = if clone_args.flags.contains(CloneFlags::CLONE_SIGHAND) { - SignalList::new_shared(&thread.signal_list) - } else { - SignalList::new_cloned(&thread.signal_list) - }; + let signal_list = + if clone_args.flags.contains(CloneFlags::CLONE_SIGHAND) { + SignalList::new_shared(&thread.signal_list) + } else { + SignalList::new_cloned(&thread.signal_list) + }; + + let mut tls = clone_args.tls.clone(); + + // Respect `clone_args`'s TLS if given. Otherwise, clone from the thread. + if tls.is_none() { + tls = thread.inner.lock().tls.clone(); + } Ok(self .files(files) .fs_context(fs_context) .signal_list(signal_list) - .name(inner.name.clone()) - .tls(clone_args.tls.clone()) + .name(thread.inner.lock().name.clone()) + .tls(tls) .set_child_tid(clone_args.set_tid_ptr) .clear_child_tid(clone_args.clear_tid_ptr) .trap_ctx(trap_ctx) @@ -246,6 +287,9 @@ impl ThreadBuilder { trap_ctx: AtomicUniqueRefCell::new(trap_ctx), fpu_state: AtomicUniqueRefCell::new(fpu_state), dead: AtomicBool::new(false), + exit_status: Spin::new(None), + all_threads_link: RBTreeAtomicLink::new(), + process_threads_link: RBTreeAtomicLink::new(), inner: Spin::new(ThreadInner { name, tls: self.tls, @@ -275,12 +319,9 @@ impl Thread { self.signal_list.raise(signal) } - /// # Safety - /// This function is unsafe because it accesses the `current_cpu()`, which needs - /// to be called in a preemption disabled context. - pub unsafe fn load_thread_area32(&self) { + pub fn activate_tls(&self) { if let Some(tls) = self.inner.lock().tls.as_ref() { - CPU::local().as_mut().set_tls32(tls); + tls.activate(); } } @@ -289,7 +330,9 @@ impl Thread { Ok(()) } - pub fn set_robust_list(&self, robust_list_address: Option<User<RobustListHead>>) { + pub fn set_robust_list( + &self, robust_list_address: Option<User<RobustListHead>>, + ) { self.inner.lock().robust_list_address = robust_list_address; } @@ -321,10 +364,7 @@ impl Thread { } pub async fn handle_syscall( - &self, - thd_alloc: ThreadAlloc<'_>, - no: usize, - args: [usize; 6], + &self, thd_alloc: ThreadAlloc<'_>, no: usize, args: [usize; 6], ) -> Option<usize> { match syscall_handlers().get(no) { Some(Some(SyscallHandler { @@ -340,18 +380,26 @@ impl Thread { } } - pub async fn force_kill(&self, signal: Signal) { - let mut proc_list = ProcessList::get().write().await; - unsafe { - // SAFETY: Preemption is disabled. - proc_list - .do_exit(self, WaitType::Signaled(signal), false) - .await; + pub fn exit(&self, exit_status: WaitType) { + { + let mut self_status = self.exit_status.lock(); + if self_status.is_some() { + // Someone has got here before us. + return; + } + + *self_status = Some(exit_status); } + + self.dead.store(true, Ordering::Release); + } + + pub fn force_kill(&self, signal: Signal) { + self.exit(WaitType::Signaled(signal)); } pub fn is_dead(&self) -> bool { - self.dead.load(Ordering::SeqCst) + self.dead.load(Ordering::Acquire) } async fn real_run(&self) { @@ -371,7 +419,10 @@ impl Thread { while !self.is_dead() { if self.signal_list.has_pending_signal() { self.signal_list - .handle(&mut self.trap_ctx.borrow(), &mut self.fpu_state.borrow()) + .handle( + &mut self.trap_ctx.borrow(), + &mut self.fpu_state.borrow(), + ) .await; } @@ -394,8 +445,14 @@ impl Thread { error_code, address: addr, }) => { + if self.is_dead() { + return; + } + let mms = &self.process.mm_list; - if let Err(signal) = mms.handle_user_page_fault(addr, error_code).await { + if let Err(signal) = + mms.handle_user_page_fault(addr, error_code).await + { self.signal_list.raise(signal); } } @@ -405,8 +462,12 @@ impl Thread { TrapType::Fault(Fault::InvalidOp) => { self.signal_list.raise(Signal::SIGILL); } - TrapType::Fault(Fault::Unknown(_)) => unimplemented!("Unhandled fault"), - TrapType::Breakpoint => unimplemented!("Breakpoint in user space"), + TrapType::Fault(Fault::Unknown(_)) => { + unimplemented!("Unhandled fault") + } + TrapType::Breakpoint => { + unimplemented!("Breakpoint in user space") + } TrapType::Irq { callback } => callback(default_irq_handler), TrapType::Timer { callback } => { callback(timer_interrupt); @@ -416,11 +477,20 @@ impl Thread { } } TrapType::Syscall { no, args } => { - if let Some(retval) = self.handle_syscall(thd_alloc, no, args).await { + if self.is_dead() { + return; + } + + if let Some(retval) = + self.handle_syscall(thd_alloc, no, args).await + { let mut trap_ctx = self.trap_ctx.borrow(); trap_ctx.set_user_return_value(retval); - #[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] + #[cfg(any( + target_arch = "riscv64", + target_arch = "loongarch64" + ))] { let pc = trap_ctx.get_program_counter(); trap_ctx.set_program_counter(pc + 4); @@ -442,14 +512,7 @@ impl Thread { CURRENT_THREAD.set(NonNull::new(&raw const *self as *mut _)); - unsafe { - eonix_preempt::disable(); - - // SAFETY: Preemption is disabled. - self.load_thread_area32(); - - eonix_preempt::enable(); - } + self.activate_tls(); let result = future.as_mut().poll(cx); @@ -463,14 +526,26 @@ impl Thread { } pub fn run(self: Arc<Thread>) -> impl Future<Output = ()> + Send + 'static { - async move { self.contexted(stackful(self.real_run())).await } + async move { + self.contexted(async { + stackful(self.real_run()).await; + + futex_exit(&self).await; + }) + .await; + + assert!( + self.is_dead(), + "`real_run` returned before the thread die?" + ); + ProcessList::send_to_reaper(self); + } } } unsafe impl Allocator for ThreadAlloc<'_> { fn allocate( - &self, - layout: core::alloc::Layout, + &self, layout: core::alloc::Layout, ) -> Result<NonNull<[u8]>, alloc::alloc::AllocError> { self.0.allocate(layout) } @@ -488,7 +563,9 @@ pub async fn yield_now() { impl Future for Yield { type Output = (); - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + fn poll( + mut self: Pin<&mut Self>, cx: &mut Context<'_>, + ) -> Poll<Self::Output> { if self.as_mut().yielded { Poll::Ready(()) } else { diff --git a/src/kernel/task/user_tls/mod.rs b/src/kernel/task/user_tls/mod.rs new file mode 100644 index 00000000..2583b580 --- /dev/null +++ b/src/kernel/task/user_tls/mod.rs @@ -0,0 +1,34 @@ +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86_64; + pub use x86_64::*; + } else { + use eonix_mm::address::VAddr; + use posix_types::ctypes::PtrT; + + use crate::prelude::KResult; + + + #[derive(Debug, Clone)] + pub struct UserTLS(VAddr); + + #[derive(Debug, Clone)] + pub struct UserTLSDescriptor(VAddr); + + impl UserTLS { + pub fn activate(&self) { + self.0; + } + } + + impl UserTLSDescriptor { + pub fn new(tp: PtrT) -> KResult<Self> { + Ok(Self(VAddr::from(tp.addr()))) + } + + pub fn read(&self) -> KResult<UserTLS> { + Ok(UserTLS(self.0)) + } + } + } +} diff --git a/src/kernel/task/user_tls/x86_64.rs b/src/kernel/task/user_tls/x86_64.rs new file mode 100644 index 00000000..5bb33b97 --- /dev/null +++ b/src/kernel/task/user_tls/x86_64.rs @@ -0,0 +1,83 @@ +use core::fmt; + +use eonix_hal::arch_exported::gdt::{GDTEntry, GDT}; +use eonix_hal::processor::CPU; +use eonix_mm::address::VAddr; +use posix_types::ctypes::PtrT; +use posix_types::x86_64::UserDescriptor; + +use crate::kernel::syscall::{User, UserMut}; +use crate::kernel::user::{CheckedUserPointer, UserPointerMut}; +use crate::prelude::KResult; + +#[derive(Debug, Clone)] +pub struct UserTLS { + desc: GDTEntry, + base: u64, +} + +pub struct UserTLSDescriptor<'a> { + ptr: UserPointerMut<'a, UserDescriptor>, +} + +impl UserTLS { + fn new(base: u32, limit: u32) -> Self { + Self { + desc: GDTEntry::new_tls(base, limit), + base: base as u64, + } + } + + fn new_page_limit(base: u32, limit_in_pages: u32) -> Self { + Self { + desc: GDTEntry::new_tls_page_limit(base, limit_in_pages), + base: base as u64, + } + } + + pub fn activate(&self) { + CPU::local().as_mut().set_tls32(self.desc, self.base); + } +} + +impl UserTLSDescriptor<'_> { + pub fn new(raw_tls: PtrT) -> KResult<Self> { + Ok(Self { + ptr: UserPointerMut::new(UserMut::<UserDescriptor>::with_addr(raw_tls.addr()))?, + }) + } + + pub fn read(&self) -> KResult<UserTLS> { + let mut desc = self.ptr.read()?; + + let base = VAddr::from(desc.base as usize); + + // Clear the TLS area if it is not present. + if desc.flags.is_read_exec_only() && !desc.flags.is_present() { + if desc.limit != 0 && base != VAddr::NULL { + let len = if desc.flags.is_limit_in_pages() { + (desc.limit as usize) << 12 + } else { + desc.limit as usize + }; + + CheckedUserPointer::new(User::new(base), len)?.zero()?; + } + } + + desc.entry = GDT::TLS32_INDEX as u32; + self.ptr.write(desc)?; + + Ok(if desc.flags.is_limit_in_pages() { + UserTLS::new_page_limit(desc.base, desc.limit) + } else { + UserTLS::new(desc.base, desc.limit) + }) + } +} + +impl fmt::Debug for UserTLSDescriptor<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("UserTLSDescriptor").finish_non_exhaustive() + } +} diff --git a/src/kernel/terminal.rs b/src/kernel/terminal.rs index 734655b7..ddc3cc1f 100644 --- a/src/kernel/terminal.rs +++ b/src/kernel/terminal.rs @@ -1,18 +1,19 @@ -use super::{ - task::{ProcessList, Session, Thread}, - user::{UserPointer, UserPointerMut}, -}; -use crate::kernel::constants::{EINTR, ENOTTY, EPERM}; -use crate::{io::Buffer, prelude::*, sync::CondVar}; -use alloc::{ - collections::vec_deque::VecDeque, - sync::{Arc, Weak}, -}; +use alloc::collections::vec_deque::VecDeque; +use alloc::sync::{Arc, Weak}; + use bitflags::bitflags; use eonix_log::ConsoleWrite; -use eonix_sync::{AsProof as _, Mutex}; +use eonix_sync::{Mutex, Proof}; use posix_types::signal::Signal; +use super::constants::ESRCH; +use super::task::{ProcessList, Session, Thread}; +use super::user::{UserPointer, UserPointerMut}; +use crate::io::Buffer; +use crate::kernel::constants::{EINTR, ENOTTY, EPERM}; +use crate::prelude::*; +use crate::sync::CondVar; + const BUFFER_SIZE: usize = 4096; const NCCS: usize = 19; @@ -351,12 +352,12 @@ pub trait TerminalDevice: Send + Sync { struct TerminalInner { termio: Termios, - session: Weak<Session>, buffer: VecDeque<u8>, } pub struct Terminal { inner: Mutex<TerminalInner>, + session: Spin<Weak<Session>>, device: Arc<dyn TerminalDevice>, cv: CondVar, } @@ -400,9 +401,9 @@ impl Terminal { Arc::new(Self { inner: Mutex::new(TerminalInner { termio: Termios::new_standard(), - session: Weak::new(), buffer: VecDeque::with_capacity(BUFFER_SIZE), }), + session: Spin::new(Weak::new()), cv: CondVar::new(), device, }) @@ -447,15 +448,21 @@ impl Terminal { } async fn signal(&self, inner: &mut TerminalInner, signal: Signal) { - if let Some(session) = inner.session.upgrade() { + if let Some(session) = self.session() { session.raise_foreground(signal).await; } + if !inner.termio.noflsh() { self.clear_read_buffer(inner); } } - async fn echo_and_signal(&self, inner: &mut TerminalInner, ch: u8, signal: Signal) { + async fn echo_and_signal( + &self, + inner: &mut TerminalInner, + ch: u8, + signal: Signal, + ) { self.echo_char(inner, ch); self.signal(inner, signal).await; } @@ -481,13 +488,19 @@ impl Terminal { match ch { 0xff => {} ch if ch == inner.termio.vintr() => { - return self.echo_and_signal(&mut inner, ch, Signal::SIGINT).await + return self + .echo_and_signal(&mut inner, ch, Signal::SIGINT) + .await } ch if ch == inner.termio.vquit() => { - return self.echo_and_signal(&mut inner, ch, Signal::SIGQUIT).await + return self + .echo_and_signal(&mut inner, ch, Signal::SIGQUIT) + .await } ch if ch == inner.termio.vsusp() => { - return self.echo_and_signal(&mut inner, ch, Signal::SIGTSTP).await + return self + .echo_and_signal(&mut inner, ch, Signal::SIGTSTP) + .await } _ => {} } @@ -517,8 +530,12 @@ impl Terminal { match ch { b'\r' if inner.termio.igncr() => {} - b'\r' if inner.termio.icrnl() => return self.do_commit_char(&mut inner, b'\n'), - b'\n' if inner.termio.inlcr() => return self.do_commit_char(&mut inner, b'\r'), + b'\r' if inner.termio.icrnl() => { + return self.do_commit_char(&mut inner, b'\n') + } + b'\n' if inner.termio.inlcr() => { + return self.do_commit_char(&mut inner, b'\r') + } _ => self.do_commit_char(&mut inner, ch), } } @@ -589,26 +606,30 @@ impl Terminal { pub async fn ioctl(&self, request: TerminalIORequest<'_>) -> KResult<()> { match request { TerminalIORequest::GetProcessGroup(pgid_pointer) => { - if let Some(session) = self.inner.lock().await.session.upgrade() { - if let Some(pgroup) = session.foreground().await { - return pgid_pointer.write(pgroup.pgid); - } - } + let Some(session) = self.session() else { + return Err(ENOTTY); + }; + + let Some(pgroup) = session.foreground_pgroup() else { + return Err(ENOTTY); + }; - Err(ENOTTY) + pgid_pointer.write(pgroup.pgid) } TerminalIORequest::SetProcessGroup(pgid) => { let pgid = pgid.read()?; let procs = ProcessList::get().read().await; - let inner = self.inner.lock().await; - let session = inner.session.upgrade(); + let Some(session) = self.session() else { + return Err(ENOTTY); + }; - if let Some(session) = session { - session.set_foreground_pgid(pgid, procs.prove()).await - } else { - Err(ENOTTY) - } + let Some(pgroup) = procs.try_find_pgroup(pgid) else { + return Err(ESRCH); + }; + + session.set_foreground_pgroup(Some(&pgroup))?; + Ok(()) } TerminalIORequest::GetWindowSize(ptr) => { // TODO: Get the actual window size @@ -630,9 +651,12 @@ impl Terminal { let mut inner = self.inner.lock().await; // TODO: We ignore unknown bits for now. - inner.termio.iflag = TermioIFlags::from_bits_truncate(user_termios.iflag as u16); - inner.termio.oflag = TermioOFlags::from_bits_truncate(user_termios.oflag as u16); - inner.termio.lflag = TermioLFlags::from_bits_truncate(user_termios.lflag as u16); + inner.termio.iflag = + TermioIFlags::from_bits_truncate(user_termios.iflag as u16); + inner.termio.oflag = + TermioOFlags::from_bits_truncate(user_termios.oflag as u16); + inner.termio.lflag = + TermioLFlags::from_bits_truncate(user_termios.lflag as u16); inner.termio.cflag = user_termios.cflag; inner.termio.line = user_termios.line; inner.termio.cc = user_termios.cc; @@ -642,30 +666,52 @@ impl Terminal { } } - /// Assign the `session` to this terminal. Drop the previous session if `forced` is true. - pub async fn set_session(&self, session: &Arc<Session>, forced: bool) -> KResult<()> { - let mut inner = self.inner.lock().await; - if let Some(session) = inner.session.upgrade() { + pub fn session(&self) -> Option<Arc<Session>> { + self.session.lock().upgrade() + } + + /// Drop our current controlled session. The old session lose its controlling + /// terminal and all processes in it will receive a SIGHUP and then SIGCONT. + pub fn drop_session(&self, procs: Proof<'_, ProcessList>) { + let session = + core::mem::replace(&mut *self.session.lock(), Weak::new()); + let Some(old_session) = session.upgrade() else { + return; + }; + + old_session._drop_control_terminal(procs); + } + + /// Assign the `session` to this terminal. + /// Drop the previous session if `forced` is true. + pub async fn set_session( + self: &Arc<Self>, + session: &Arc<Session>, + forced: bool, + procs: Proof<'_, ProcessList>, + ) -> KResult<()> { + let mut cur_session = self.session.lock(); + + // XXX: Holding spinlock for too long? + if let Some(old_session) = cur_session.upgrade() { + if old_session.sid == session.sid { + return Ok(()); + } + if !forced { - Err(EPERM) - } else { - session.drop_control_terminal().await; - inner.session = Arc::downgrade(&session); - Ok(()) + return Err(EPERM); } - } else { - // Sessions should set their `control_terminal` field. - inner.session = Arc::downgrade(&session); - Ok(()) + + // TODO: Check whether the caller has the CAP_SYS_ADMIN capability. + + // We've stolen the terminal from the old session. + old_session._drop_control_terminal(procs); } - } - pub async fn drop_session(&self) { - self.inner.lock().await.session = Weak::new(); - } + *cur_session = Arc::downgrade(session); + session._set_control_terminal(self, procs); - pub async fn session(&self) -> Option<Arc<Session>> { - self.inner.lock().await.session.upgrade() + Ok(()) } } diff --git a/src/kernel/timer.rs b/src/kernel/timer.rs index 9b6a3ff2..1dbb1382 100644 --- a/src/kernel/timer.rs +++ b/src/kernel/timer.rs @@ -76,6 +76,8 @@ impl Ticks { } impl Instant { + pub const UNIX_EPOCH: Self = Self::default(); + pub const fn default() -> Self { Instant { secs_since_epoch: 0, diff --git a/src/kernel/user/dataflow.rs b/src/kernel/user/dataflow.rs index 02e7d791..5d8ac167 100644 --- a/src/kernel/user/dataflow.rs +++ b/src/kernel/user/dataflow.rs @@ -1,18 +1,15 @@ -use crate::{ - io::{Buffer, FillResult}, - prelude::*, -}; -use crate::{ - io::{IntoStream, Stream}, - kernel::{ - constants::{EFAULT, EINVAL}, - syscall::{User, UserMut}, - }, -}; -use core::{arch::asm, ffi::CStr, marker::PhantomData}; +use core::arch::asm; +use core::ffi::CStr; +use core::marker::PhantomData; + use eonix_mm::address::Addr; use eonix_preempt::assert_preempt_enabled; +use crate::io::{Buffer, FillResult, IntoStream, Stream}; +use crate::kernel::constants::{EFAULT, EINVAL}; +use crate::kernel::syscall::{User, UserMut}; +use crate::prelude::*; + pub struct CheckedUserPointer<'a> { ptr: User<u8>, len: usize, diff --git a/src/kernel/vfs/dentry.rs b/src/kernel/vfs/dentry.rs index 8bcd9f8a..7741c55a 100644 --- a/src/kernel/vfs/dentry.rs +++ b/src/kernel/vfs/dentry.rs @@ -1,35 +1,62 @@ pub mod dcache; +mod walk; -use super::{ - inode::{Ino, Inode, Mode, RenameData, WriteOffset}, - DevId, FsContext, -}; -use crate::{ - hash::KernelHasher, - io::{Buffer, ByteBuffer}, - kernel::{block::BlockDevice, CharDevice}, - path::{Path, PathComponent}, - prelude::*, - rcu::{RCUNode, RCUPointer, RCUReadGuard}, -}; -use crate::{ - io::Stream, - kernel::constants::{EEXIST, EINVAL, EIO, EISDIR, ELOOP, ENOENT, ENOTDIR, EPERM, ERANGE}, -}; -use alloc::sync::{Arc, Weak}; -use core::{ - fmt, - hash::{BuildHasher, BuildHasherDefault, Hasher}, - ops::ControlFlow, - sync::atomic::{AtomicPtr, AtomicU64, Ordering}, -}; +use alloc::sync::Arc; +use core::cell::UnsafeCell; +use core::fmt; +use core::hash::{BuildHasher, BuildHasherDefault, Hasher}; +use core::sync::atomic::{AtomicPtr, AtomicU64, AtomicU8, Ordering}; + +use arcref::AsArcRef; use eonix_sync::LazyLock; use pointers::BorrowedArc; -use posix_types::{namei::RenameFlags, open::OpenFlags, result::PosixError, stat::StatX}; +use posix_types::namei::RenameFlags; +use posix_types::open::OpenFlags; +use posix_types::result::PosixError; +use posix_types::stat::StatX; + +use super::inode::{Ino, InodeUse, RenameData, WriteOffset}; +use super::types::{DeviceId, Format, Mode, Permission}; +use super::FsContext; +use crate::hash::KernelHasher; +use crate::io::{Buffer, Stream}; +use crate::kernel::block::BlockDevice; +use crate::kernel::constants::{ + EEXIST, EINVAL, EISDIR, ELOOP, ENOENT, EPERM, ERANGE, +}; +use crate::kernel::CharDevice; +use crate::path::Path; +use crate::prelude::*; +use crate::rcu::{rcu_read_lock, RCUNode, RCUPointer, RCUReadGuard}; + +// TODO: Implement slab reclaim +#[allow(unused)] +const D_INVALID: u8 = 0; +const D_REGULAR: u8 = 1; +const D_DIRECTORY: u8 = 2; +const D_SYMLINK: u8 = 3; + +#[derive(Debug, PartialEq, Eq)] +enum DentryKind { + Regular = D_REGULAR as isize, + Directory = D_DIRECTORY as isize, + Symlink = D_SYMLINK as isize, +} -struct DentryData { - inode: Arc<dyn Inode>, - flags: u64, +/// The [`Inode`] associated with a [`Dentry`]. +/// +/// We could assign an inode to a negative dentry exactly once when the dentry +/// is invalid and we create a file or directory on it, or the dentry is brought +/// to the dcache by [lookup()]. +/// +/// This guarantees that as long as we acquire a non-invalid from [`Self::kind`], +/// we are synced with the writer and can safely read the [`Self::inode`] field +/// without reading torn data. +/// +/// [lookup()]: crate::kernel::vfs::inode::InodeDirOps::lookup +struct AssociatedInode { + kind: UnsafeCell<Option<DentryKind>>, + inode: UnsafeCell<Option<InodeUse>>, } /// # Safety @@ -48,8 +75,7 @@ pub struct Dentry { prev: AtomicPtr<Dentry>, next: AtomicPtr<Dentry>, - // RCU Mutable - data: RCUPointer<DentryData>, + inode: AssociatedInode, } pub(super) static DROOT: LazyLock<Arc<Dentry>> = LazyLock::new(|| { @@ -59,7 +85,7 @@ pub(super) static DROOT: LazyLock<Arc<Dentry>> = LazyLock::new(|| { hash: AtomicU64::new(0), prev: AtomicPtr::default(), next: AtomicPtr::default(), - data: RCUPointer::empty(), + inode: AssociatedInode::new(), }); unsafe { @@ -79,12 +105,6 @@ impl fmt::Debug for Dentry { } } -const D_DIRECTORY: u64 = 1; -#[allow(dead_code)] -const D_MOUNTPOINT: u64 = 2; -const D_SYMLINK: u64 = 4; -const D_REGULAR: u64 = 8; - impl RCUNode<Dentry> for Dentry { fn rcu_prev(&self) -> &AtomicPtr<Self> { &self.prev @@ -115,50 +135,19 @@ impl Dentry { self.hash.store(hash, Ordering::Relaxed); } - - fn find(self: &Arc<Self>, name: &[u8]) -> KResult<Arc<Self>> { - let data = self.data.load(); - let data = data.as_ref().ok_or(ENOENT)?; - - if data.flags & D_DIRECTORY == 0 { - return Err(ENOTDIR); - } - - match name { - b"." => Ok(self.clone()), - b".." => Ok(self.parent().clone()), - _ => { - let dentry = Dentry::create(self.clone(), name); - - if let Some(found) = dcache::d_find_fast(&dentry) { - unsafe { - // SAFETY: This is safe because the dentry is never shared with - // others so we can drop them safely. - let _ = dentry.name.swap(None); - let _ = dentry.parent.swap(None); - } - - return Ok(found); - } - - dcache::d_try_revalidate(&dentry); - dcache::d_add(dentry.clone()); - - Ok(dentry) - } - } - } } impl Dentry { pub fn create(parent: Arc<Dentry>, name: &[u8]) -> Arc<Self> { + // TODO!!!: don't acquire our parent's refcount here... + let val = Arc::new(Self { parent: RCUPointer::new(parent), name: RCUPointer::new(Arc::new(Arc::from(name))), hash: AtomicU64::new(0), prev: AtomicPtr::default(), next: AtomicPtr::default(), - data: RCUPointer::empty(), + inode: AssociatedInode::new(), }); val.rehash(); @@ -174,7 +163,7 @@ impl Dentry { && &***self.name() == &***other.name() } - pub fn name(&self) -> RCUReadGuard<BorrowedArc<Arc<[u8]>>> { + pub fn name(&self) -> RCUReadGuard<'_, BorrowedArc<'_, Arc<[u8]>>> { self.name.load().expect("Dentry has no name") } @@ -182,7 +171,7 @@ impl Dentry { (***self.name()).clone() } - pub fn parent<'a>(&self) -> RCUReadGuard<'a, BorrowedArc<Dentry>> { + pub fn parent<'a>(&self) -> RCUReadGuard<'a, BorrowedArc<'_, Dentry>> { self.parent.load().expect("Dentry has no parent") } @@ -192,205 +181,96 @@ impl Dentry { .map_or(core::ptr::null(), |parent| Arc::as_ptr(&parent)) } - fn save_data(&self, inode: Arc<dyn Inode>, flags: u64) -> KResult<()> { - let new = DentryData { inode, flags }; - - // TODO!!!: We don't actually need to use `RCUPointer` here - // Safety: this function may only be called from `create`-like functions which requires the - // superblock's write locks to be held, so only one creation can happen at a time and we - // can't get a reference to the old data. - let old = unsafe { self.data.swap(Some(Arc::new(new))) }; - assert!(old.is_none()); - - Ok(()) - } - - pub fn save_reg(&self, file: Arc<dyn Inode>) -> KResult<()> { - self.save_data(file, D_REGULAR) - } - - pub fn save_symlink(&self, link: Arc<dyn Inode>) -> KResult<()> { - self.save_data(link, D_SYMLINK) + pub fn fill(&self, file: InodeUse) { + self.inode.store(file); } - pub fn save_dir(&self, dir: Arc<dyn Inode>) -> KResult<()> { - self.save_data(dir, D_DIRECTORY) + pub fn inode(&self) -> Option<InodeUse> { + self.inode.load().map(|(_, inode)| inode.clone()) } - pub fn get_inode(&self) -> KResult<Arc<dyn Inode>> { - self.data - .load() - .as_ref() - .ok_or(ENOENT) - .map(|data| data.inode.clone()) + pub fn get_inode(&self) -> KResult<InodeUse> { + self.inode().ok_or(ENOENT) } pub fn is_directory(&self) -> bool { - let data = self.data.load(); - data.as_ref() - .map_or(false, |data| data.flags & D_DIRECTORY != 0) + self.inode + .load() + .map_or(false, |(kind, _)| kind == DentryKind::Directory) } pub fn is_valid(&self) -> bool { - self.data.load().is_some() + self.inode.load().is_some() } - pub fn open_check(self: &Arc<Self>, flags: OpenFlags, mode: Mode) -> KResult<()> { - let data = self.data.load(); - - if data.is_some() { - if flags.contains(OpenFlags::O_CREAT | OpenFlags::O_EXCL) { - Err(EEXIST) - } else { - Ok(()) - } - } else { - if !flags.contains(OpenFlags::O_CREAT) { - return Err(ENOENT); + pub async fn open_check( + self: &Arc<Self>, flags: OpenFlags, perm: Permission, + ) -> KResult<()> { + match self.inode.load() { + Some(_) => { + if flags.contains(OpenFlags::O_CREAT | OpenFlags::O_EXCL) { + Err(EEXIST) + } else { + Ok(()) + } } + None => { + if !flags.contains(OpenFlags::O_CREAT) { + return Err(ENOENT); + } - let parent = self.parent().get_inode()?; - parent.creat(self, mode) + let parent = self.parent().get_inode()?; + parent.create(self, perm).await + } } } } impl Dentry { - fn resolve_directory( - context: &FsContext, - dentry: Arc<Self>, - nrecur: u32, + pub async fn open( + context: &FsContext, path: &Path, follow_symlinks: bool, ) -> KResult<Arc<Self>> { - if nrecur >= 16 { - return Err(ELOOP); - } - - let data = dentry.data.load(); - let data = data.as_ref().ok_or(ENOENT)?; - - match data.flags { - flags if flags & D_REGULAR != 0 => Err(ENOTDIR), - flags if flags & D_DIRECTORY != 0 => Ok(dentry), - flags if flags & D_SYMLINK != 0 => { - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); - - data.inode.readlink(&mut buffer)?; - let path = Path::new(buffer.data())?; - - let dentry = - Self::open_recursive(context, &dentry.parent(), path, true, nrecur + 1)?; - - Self::resolve_directory(context, dentry, nrecur + 1) - } - _ => panic!("Invalid dentry flags"), - } + let cwd = context.cwd.lock().clone(); + Self::open_at(context, &cwd, path, follow_symlinks).await } - pub fn open_recursive( - context: &FsContext, - cwd: &Arc<Self>, - path: Path, - follow: bool, - nrecur: u32, + pub async fn open_at( + context: &FsContext, at: &Arc<Self>, path: &Path, follow_symlinks: bool, ) -> KResult<Arc<Self>> { - // too many recursive search layers will cause stack overflow - // so we use 16 for now - if nrecur >= 16 { - return Err(ELOOP); - } - - let mut cwd = if path.is_absolute() { - context.fsroot.clone() - } else { - cwd.clone() - }; + let mut found = context.start_recursive_walk(at, path).await?; - for item in path.iter() { - if let PathComponent::TrailingEmpty = item { - if cwd.data.load().as_ref().is_none() { - return Ok(cwd); - } - } - - cwd = Self::resolve_directory(context, cwd, nrecur)?; - - match item { - PathComponent::TrailingEmpty | PathComponent::Current => {} // pass - PathComponent::Parent => { - if !cwd.hash_eq(&context.fsroot) { - let parent = cwd.parent().clone(); - cwd = Self::resolve_directory(context, parent, nrecur)?; - } - continue; - } - PathComponent::Name(name) => { - cwd = cwd.find(name)?; - } - } + if !follow_symlinks { + return Ok(found); } - if follow { - let data = cwd.data.load(); - - if let Some(data) = data.as_ref() { - if data.flags & D_SYMLINK != 0 { - let data = cwd.data.load(); - let data = data.as_ref().unwrap(); - let mut buffer = [0u8; 256]; - let mut buffer = ByteBuffer::new(&mut buffer); + loop { + let Some((DentryKind::Symlink, inode)) = found.inode.load() else { + return Ok(found); + }; - data.inode.readlink(&mut buffer)?; - let path = Path::new(buffer.data())?; - - let parent = cwd.parent().clone(); - cwd = Self::open_recursive(context, &parent, path, true, nrecur + 1)?; - } - } + found = context.follow_symlink(found.aref(), inode, 0).await?; } - - Ok(cwd) - } - - pub fn open(context: &FsContext, path: Path, follow_symlinks: bool) -> KResult<Arc<Self>> { - let cwd = context.cwd.lock().clone(); - Dentry::open_recursive(context, &cwd, path, follow_symlinks, 0) - } - - pub fn open_at( - context: &FsContext, - at: &Arc<Self>, - path: Path, - follow_symlinks: bool, - ) -> KResult<Arc<Self>> { - Dentry::open_recursive(context, at, path, follow_symlinks, 0) } pub fn get_path( - self: &Arc<Dentry>, - context: &FsContext, - buffer: &mut dyn Buffer, + self: &Arc<Self>, context: &FsContext, buffer: &mut dyn Buffer, ) -> KResult<()> { - let locked_parent = self.parent(); + let rcu_read = rcu_read_lock(); - let path = { - let mut path = vec![]; + let mut path = vec![]; - let mut parent = locked_parent.borrow(); - let mut dentry = BorrowedArc::new(self); + let mut current = self.aref(); + let mut parent = self.parent.dereference(&rcu_read).unwrap(); - while Arc::as_ptr(&dentry) != Arc::as_ptr(&context.fsroot) { - if path.len() > 32 { - return Err(ELOOP); - } - - path.push(dentry.name().clone()); - dentry = parent; - parent = dentry.parent.load_protected(&locked_parent).unwrap(); + while !current.ptr_eq_arc(&context.fsroot) { + if path.len() > 32 { + return Err(ELOOP); } - path - }; + path.push(current.name.dereference(&rcu_read).unwrap()); + current = parent; + parent = current.parent.dereference(&rcu_read).unwrap(); + } buffer.fill(b"/")?.ok_or(ERANGE)?; for item in path.iter().rev().map(|name| name.as_ref()) { @@ -405,51 +285,59 @@ impl Dentry { } impl Dentry { - pub fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + pub async fn read( + &self, buffer: &mut dyn Buffer, offset: usize, + ) -> KResult<usize> { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.mode.load().format() { - Mode::DIR => Err(EISDIR), - Mode::REG => inode.read(buffer, offset), - Mode::BLK => { + match inode.format { + Format::DIR => Err(EISDIR), + Format::REG => inode.read(buffer, offset).await, + Format::BLK => { let device = BlockDevice::get(inode.devid()?)?; - Ok(device.read_some(offset, buffer)?.allow_partial()) + Ok(device.read_some(offset, buffer).await?.allow_partial()) } - Mode::CHR => { + Format::CHR => { let device = CharDevice::get(inode.devid()?).ok_or(EPERM)?; - device.read(buffer) + device.read(buffer).await } _ => Err(EINVAL), } } - pub fn write(&self, stream: &mut dyn Stream, offset: WriteOffset) -> KResult<usize> { + pub async fn write( + &self, stream: &mut dyn Stream, offset: WriteOffset<'_>, + ) -> KResult<usize> { let inode = self.get_inode()?; // Safety: Changing mode alone will have no effect on the file's contents - match inode.mode.load().format() { - Mode::DIR => Err(EISDIR), - Mode::REG => inode.write(stream, offset), - Mode::BLK => Err(EINVAL), // TODO - Mode::CHR => CharDevice::get(inode.devid()?).ok_or(EPERM)?.write(stream), + match inode.format { + Format::DIR => Err(EISDIR), + Format::REG => inode.write(stream, offset).await, + Format::BLK => Err(EINVAL), // TODO + Format::CHR => { + CharDevice::get(inode.devid()?).ok_or(EPERM)?.write(stream) + } _ => Err(EINVAL), } } - pub fn readdir<F>(&self, offset: usize, mut callback: F) -> KResult<usize> + pub async fn readdir<F>( + &self, offset: usize, mut for_each_entry: F, + ) -> KResult<KResult<usize>> where - F: FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, + F: FnMut(&[u8], Ino) -> KResult<bool> + Send, { let dir = self.get_inode()?; - dir.do_readdir(offset, &mut callback) + dir.readdir(offset, &mut for_each_entry).await } - pub fn mkdir(&self, mode: Mode) -> KResult<()> { + pub async fn mkdir(&self, perm: Permission) -> KResult<()> { if self.get_inode().is_ok() { Err(EEXIST) } else { let dir = self.parent().get_inode()?; - dir.mkdir(self, mode) + dir.mkdir(self, perm).await } } @@ -457,50 +345,52 @@ impl Dentry { self.get_inode()?.statx(stat, mask) } - pub fn truncate(&self, size: usize) -> KResult<()> { - self.get_inode()?.truncate(size) + pub async fn truncate(&self, size: usize) -> KResult<()> { + self.get_inode()?.truncate(size).await } - pub fn unlink(self: &Arc<Self>) -> KResult<()> { + pub async fn unlink(self: &Arc<Self>) -> KResult<()> { if self.get_inode().is_err() { Err(ENOENT) } else { let dir = self.parent().get_inode()?; - dir.unlink(self) + dir.unlink(self).await } } - pub fn symlink(self: &Arc<Self>, link: &[u8]) -> KResult<()> { + pub async fn symlink(self: &Arc<Self>, link: &[u8]) -> KResult<()> { if self.get_inode().is_ok() { Err(EEXIST) } else { let dir = self.parent().get_inode()?; - dir.symlink(self, link) + dir.symlink(self, link).await } } - pub fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { - self.get_inode()?.readlink(buffer) + pub async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { + self.get_inode()?.readlink(buffer).await } - pub fn mknod(&self, mode: Mode, devid: DevId) -> KResult<()> { + pub async fn mknod(&self, mode: Mode, devid: DeviceId) -> KResult<()> { if self.get_inode().is_ok() { Err(EEXIST) } else { let dir = self.parent().get_inode()?; - dir.mknod(self, mode, devid) + dir.mknod(self, mode, devid).await } } - pub fn chmod(&self, mode: Mode) -> KResult<()> { - self.get_inode()?.chmod(mode) + pub async fn chmod(&self, mode: Mode) -> KResult<()> { + self.get_inode()?.chmod(mode.perm()).await } - pub fn chown(&self, uid: u32, gid: u32) -> KResult<()> { - self.get_inode()?.chown(uid, gid) + pub async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { + self.get_inode()?.chown(uid, gid).await } - pub fn rename(self: &Arc<Self>, new: &Arc<Self>, flags: RenameFlags) -> KResult<()> { + pub async fn rename( + self: &Arc<Self>, new: &Arc<Self>, flags: RenameFlags, + ) -> KResult<()> { if Arc::ptr_eq(self, new) { return Ok(()); } @@ -509,22 +399,92 @@ impl Dentry { let new_parent = new.parent().get_inode()?; // If the two dentries are not in the same filesystem, return EXDEV. - if !Weak::ptr_eq(&old_parent.vfs, &new_parent.vfs) { + if old_parent.sbref().eq(&new_parent.sbref()) { Err(PosixError::EXDEV)?; } - let vfs = old_parent.vfs.upgrade().ok_or(EIO)?; - let rename_data = RenameData { old_dentry: self, new_dentry: new, new_parent, - vfs, is_exchange: flags.contains(RenameFlags::RENAME_EXCHANGE), no_replace: flags.contains(RenameFlags::RENAME_NOREPLACE), }; // Delegate to the parent directory's rename implementation - old_parent.rename(rename_data) + old_parent.rename(rename_data).await } } + +impl DentryKind { + fn into_raw(self) -> u8 { + unsafe { core::mem::transmute(self) } + } + + fn from_raw(raw: u8) -> Option<Self> { + unsafe { core::mem::transmute(raw) } + } + + fn as_atomic(me: &UnsafeCell<Option<Self>>) -> &AtomicU8 { + unsafe { AtomicU8::from_ptr(me.get().cast()) } + } + + fn atomic_acq(me: &UnsafeCell<Option<Self>>) -> Option<Self> { + Self::from_raw(Self::as_atomic(me).load(Ordering::Acquire)) + } + + fn atomic_swap_acqrel( + me: &UnsafeCell<Option<Self>>, kind: Option<Self>, + ) -> Option<Self> { + Self::from_raw( + Self::as_atomic(me) + .swap(kind.map_or(0, Self::into_raw), Ordering::AcqRel), + ) + } +} + +impl AssociatedInode { + fn new() -> Self { + Self { + inode: UnsafeCell::new(None), + kind: UnsafeCell::new(None), + } + } + + fn store(&self, inode: InodeUse) { + let kind = match inode.format { + Format::REG | Format::BLK | Format::CHR => DentryKind::Regular, + Format::DIR => DentryKind::Directory, + Format::LNK => DentryKind::Symlink, + }; + + unsafe { + // SAFETY: We should be the first and only one to store the inode as + // is checked below. All other readers reading non-invalid + // kind will see the fully written inode. + self.inode.get().write(Some(inode)); + } + + assert_eq!( + DentryKind::atomic_swap_acqrel(&self.kind, Some(kind)), + None, + "Dentry can only be stored once." + ); + } + + fn kind(&self) -> Option<DentryKind> { + DentryKind::atomic_acq(&self.kind) + } + + fn load(&self) -> Option<(DentryKind, &InodeUse)> { + self.kind().map(|kind| unsafe { + let inode = (&*self.inode.get()) + .as_ref() + .expect("Dentry with non-invalid kind has no inode"); + (kind, inode) + }) + } +} + +unsafe impl Send for AssociatedInode {} +unsafe impl Sync for AssociatedInode {} diff --git a/src/kernel/vfs/dentry/dcache.rs b/src/kernel/vfs/dentry/dcache.rs index 188a1cfc..ee7503dc 100644 --- a/src/kernel/vfs/dentry/dcache.rs +++ b/src/kernel/vfs/dentry/dcache.rs @@ -1,13 +1,13 @@ -use super::{Dentry, Inode}; +use super::Dentry; use crate::kernel::constants::ENOENT; -use crate::kernel::task::block_on; -use crate::kernel::vfs::inode::Mode; -use crate::rcu::RCUPointer; +use crate::rcu::{RCUPointer, RCUReadLock}; use crate::{ prelude::*, rcu::{RCUIterator, RCUList}, }; use alloc::sync::Arc; +use arcref::ArcRef; +use core::ops::Deref; use core::sync::atomic::Ordering; use eonix_sync::Mutex; @@ -18,50 +18,58 @@ static DCACHE: [RCUList<Dentry>; 1 << DCACHE_HASH_BITS] = static D_EXCHANGE_LOCK: Mutex<()> = Mutex::new(()); -pub fn d_hinted(dentry: &Dentry) -> &'static RCUList<Dentry> { - let hash = dentry.hash.load(Ordering::Relaxed) as usize & ((1 << DCACHE_HASH_BITS) - 1); +pub trait DCacheItem { + fn d_hash(&self) -> usize; + fn d_parent(&self) -> *const Dentry; + fn d_name<'r, 'a: 'r, 'b: 'a>( + &'a self, + rcu_read: &'b RCUReadLock, + ) -> impl Deref<Target = [u8]> + 'r; +} + +fn d_eq(lhs: &impl DCacheItem, rhs: &impl DCacheItem, rcu_read: &RCUReadLock) -> bool { + lhs.d_hash() == rhs.d_hash() + && lhs.d_parent() == rhs.d_parent() + && *lhs.d_name(rcu_read) == *rhs.d_name(rcu_read) +} + +fn d_hinted(item: &impl DCacheItem) -> &'static RCUList<Dentry> { + let hash = item.d_hash() & ((1 << DCACHE_HASH_BITS) - 1); &DCACHE[hash] } -pub fn d_iter_for(dentry: &Dentry) -> RCUIterator<'static, Dentry> { - d_hinted(dentry).iter() +fn d_iter_for<'rcu>( + item: &impl DCacheItem, + rcu_read: &'rcu RCUReadLock, +) -> RCUIterator<'static, 'rcu, Dentry> { + d_hinted(item).iter(rcu_read) +} + +pub fn d_find_rcu<'rcu>( + item: &impl DCacheItem, + rcu_read: &'rcu RCUReadLock, +) -> Option<ArcRef<'rcu, Dentry>> { + d_iter_for(item, rcu_read).find(|cur_ref| cur_ref.with_arc(|cur| d_eq(cur, item, rcu_read))) } /// Add the dentry to the dcache pub fn d_add(dentry: Arc<Dentry>) { + // TODO: Add `children` field to parent and lock parent dentry to avoid + // concurrent insertion causing duplication. d_hinted(&dentry).insert(dentry); } -pub fn d_find_fast(dentry: &Dentry) -> Option<Arc<Dentry>> { - d_iter_for(dentry) - .find(|cur| cur.hash_eq(dentry)) - .map(|dentry| dentry.clone()) -} - /// Call `lookup()` on the parent inode to try find if the dentry points to a valid inode /// /// Silently fail without any side effects -pub fn d_try_revalidate(dentry: &Arc<Dentry>) { - let _lock = block_on(D_EXCHANGE_LOCK.lock()); - - (|| -> KResult<()> { - let parent = dentry.parent().get_inode()?; - let inode = parent.lookup(dentry)?.ok_or(ENOENT)?; +pub async fn d_try_revalidate(dentry: &Arc<Dentry>) -> KResult<()> { + let _lock = D_EXCHANGE_LOCK.lock().await; - d_save(dentry, inode) - })() - .unwrap_or_default(); -} + let parent = dentry.parent().get_inode()?; + let inode = parent.lookup(dentry).await?.ok_or(ENOENT)?; -/// Save the inode to the dentry. -/// -/// Dentry flags will be determined by the inode's mode. -pub fn d_save(dentry: &Arc<Dentry>, inode: Arc<dyn Inode>) -> KResult<()> { - match inode.mode.load().format() { - Mode::DIR => dentry.save_dir(inode), - Mode::LNK => dentry.save_symlink(inode), - _ => dentry.save_reg(inode), - } + dentry.fill(inode); + Ok(()) } /// Replace the old dentry with the new one in the dcache @@ -95,3 +103,34 @@ pub async fn d_exchange(old: &Arc<Dentry>, new: &Arc<Dentry>) { d_add(old.clone()); d_add(new.clone()); } + +impl DCacheItem for Arc<Dentry> { + fn d_hash(&self) -> usize { + self.hash.load(Ordering::Relaxed) as usize + } + + fn d_parent(&self) -> *const Dentry { + self.parent_addr() + } + + fn d_name<'r, 'a: 'r, 'b: 'a>( + &'a self, + rcu_read: &'b RCUReadLock, + ) -> impl Deref<Target = [u8]> + 'r { + struct Name<'a>(ArcRef<'a, Arc<[u8]>>); + + impl Deref for Name<'_> { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + Name( + self.name + .dereference(rcu_read) + .expect("Dentry should have a non-null name"), + ) + } +} diff --git a/src/kernel/vfs/dentry/walk.rs b/src/kernel/vfs/dentry/walk.rs new file mode 100644 index 00000000..7b1060ac --- /dev/null +++ b/src/kernel/vfs/dentry/walk.rs @@ -0,0 +1,360 @@ +use alloc::boxed::Box; +use alloc::sync::Arc; +use core::future::Future; +use core::hash::{BuildHasher, BuildHasherDefault, Hasher}; +use core::ops::Deref; +use core::pin::Pin; + +use arcref::{ArcRef, AsArcRef}; +use posix_types::result::PosixError; + +use super::dcache::{self, DCacheItem}; +use super::{Dentry, DentryKind}; +use crate::hash::KernelHasher; +use crate::io::ByteBuffer; +use crate::kernel::constants::ELOOP; +use crate::kernel::vfs::inode::InodeUse; +use crate::kernel::vfs::FsContext; +use crate::path::{Path, PathComponent, PathIterator}; +use crate::prelude::KResult; +use crate::rcu::{rcu_read_lock, RCUReadLock}; + +struct DentryFind<'a, 'b> { + parent: &'a Dentry, + name: &'b [u8], + hash: usize, +} + +pub enum WalkResultRcu<'rcu, 'path> { + Err(PosixError), + Ok(ArcRef<'rcu, Dentry>), + Symlink { + symlink: ArcRef<'rcu, Dentry>, + inode: InodeUse, + }, + Miss { + parent: ArcRef<'rcu, Dentry>, + name: &'path [u8], + }, +} + +pub enum WalkResult { + Err(PosixError), + Ok(Arc<Dentry>), + Symlink { + symlink: Arc<Dentry>, + inode: InodeUse, + }, +} + +impl Dentry { + /// Quick path of the dentry find operation. + /// + /// Check invalid and non-directory dentries, return immediately on dot and + /// dotdot component, and do a quick rcu dcache lookup. + /// + /// Note that while `Some(dentry)` guarantees present and valid dentry, + /// returning `None` is acceptable if the actual file exists but is not in + /// the dentry cache. If so, we should check again with `lookup`. + fn find_rcu<'r, 's: 'r>( + self: ArcRef<'s, Self>, + name: &[u8], + rcu_read: &'r RCUReadLock, + ) -> Result<Option<ArcRef<'r, Self>>, PosixError> { + match self.inode.load() { + Some((DentryKind::Directory, _)) => {} + Some(_) => return Err(PosixError::ENOTDIR), + None => return Err(PosixError::ENOENT), + } + + match name { + b"." => Ok(Some(self)), + b".." => Ok(Some( + self.parent + .dereference(rcu_read) + .expect("The field `parent` should be non-null"), + )), + _ => { + let dentry_find = DentryFind::new(&self, name); + Ok(dcache::d_find_rcu(&dentry_find, rcu_read)) + } + } + } + + async fn find_slow(self: &Arc<Self>, name: &[u8]) -> Result<Arc<Self>, PosixError> { + let dentry = Dentry::create(self.clone(), name); + + let _ = dcache::d_try_revalidate(&dentry).await; + dcache::d_add(dentry.clone()); + + Ok(dentry) + } + + pub async fn find_full(self: &Arc<Self>, name: &[u8]) -> Result<Arc<Self>, PosixError> { + if let Some(dentry) = self.aref().find_rcu(name, &rcu_read_lock())? { + return Ok(dentry.clone_arc()); + } + + self.find_slow(name).await + } +} + +impl FsContext { + /// Walk the pathname and try to find the corresponding dentry FAST without + /// consulting the VFS for invalid dentries encountered. + fn walk_rcu<'rcu, 'path>( + &self, + mut current: ArcRef<'rcu, Dentry>, + iter: &mut PathIterator<'path>, + rcu_read: &'rcu RCUReadLock, + ) -> WalkResultRcu<'rcu, 'path> { + use PathComponent::*; + + loop { + let inode = current.inode.load(); + + if iter.is_empty() { + break; + } + + // Skip symlink resolution in rcu walk without consuming the iter. + if let Some((DentryKind::Symlink, inode)) = inode { + return WalkResultRcu::Symlink { + symlink: current, + inode: inode.clone(), + }; + } + + let Some(component) = iter.next() else { + break; + }; + + match (inode, component) { + // Skip trailing empty and dot for normal directories. + (Some((DentryKind::Directory, _)), TrailingEmpty | Current) => {} + // Walk to parent directory unless we are at the filesystem root. + (Some((DentryKind::Directory, _)), Parent) => { + if current.ptr_eq_arc(&self.fsroot) { + continue; + } + + current = current + .parent + .dereference(&rcu_read) + .expect("parent should exist"); + } + // Normal directory traversal + (Some((DentryKind::Directory, _)), Name(name)) => { + match current.find_rcu(name, &rcu_read) { + Err(err) => return WalkResultRcu::Err(err), + Ok(Some(found)) => { + current = found; + } + Ok(None) => { + return WalkResultRcu::Miss { + name, + parent: current, + }; + } + } + } + // Not a directory, fail and exit. + (Some(_), _) => return WalkResultRcu::Err(PosixError::ENOTDIR), + // Return invalid trailing entries directly. + (None, TrailingEmpty) => return WalkResultRcu::Ok(current), + // Invalid intermediate entries are not acceptable. + (None, _) => return WalkResultRcu::Err(PosixError::ENOENT), + } + } + + WalkResultRcu::Ok(current) + } + + /// Walk the pathname slowly with refcounts held and VFS lookups. + async fn walk_slow(&self, mut current: Arc<Dentry>, iter: &mut PathIterator<'_>) -> WalkResult { + use PathComponent::*; + + loop { + // `current` should be the parent directory and `component` is the + // next path component we are stepping into. + + if iter.is_empty() { + break; + } + + if let Some((DentryKind::Symlink, inode)) = current.inode.load() { + return WalkResult::Symlink { + inode: inode.clone(), + symlink: current, + }; + } + + let Some(component) = iter.next() else { + break; + }; + + match (current.inode.load(), &component) { + // Normal directory traversal + (Some((DentryKind::Directory, _)), _) => {} + // Not a directory, fail and exit. + (Some(_), _) => return WalkResult::Err(PosixError::ENOTDIR), + // Return invalid trailing entries directly. + (None, TrailingEmpty) => return WalkResult::Ok(current), + // Invalid intermediate entries are not acceptable. + (None, _) => return WalkResult::Err(PosixError::ENOENT), + } + + match component { + PathComponent::TrailingEmpty => {} + PathComponent::Current => {} + PathComponent::Parent => { + if current.hash_eq(&self.fsroot) { + continue; + } + + let parent = current.parent().clone(); + current = parent; + } + PathComponent::Name(name) => { + match current.find_full(name).await { + Ok(found) => current = found, + Err(err) => return WalkResult::Err(err), + }; + } + } + } + + WalkResult::Ok(current) + } + + /// Walk the pathname and get an accurate answer. Stop at symlinks. + async fn walk_full( + &self, + current: ArcRef<'_, Dentry>, + iter: &mut PathIterator<'_>, + ) -> WalkResult { + let (parent_slow, name_slow); + + match self.walk_rcu(current, iter, &rcu_read_lock()) { + WalkResultRcu::Err(error) => return WalkResult::Err(error.into()), + WalkResultRcu::Ok(dentry) => return WalkResult::Ok(dentry.clone_arc()), + WalkResultRcu::Symlink { symlink, inode } => { + return WalkResult::Symlink { + symlink: symlink.clone_arc(), + inode, + }; + } + WalkResultRcu::Miss { parent, name } => { + // Fallback to regular refcounted lookup + parent_slow = parent.clone_arc(); + name_slow = name; + } + } + + match parent_slow.find_slow(name_slow).await { + Ok(found) => self.walk_slow(found, iter).await, + Err(err) => return WalkResult::Err(err), + } + } + + pub async fn follow_symlink( + &self, + symlink: ArcRef<'_, Dentry>, + inode: &InodeUse, + nr_follows: u32, + ) -> KResult<Arc<Dentry>> { + let mut target = [0; 256]; + let mut target = ByteBuffer::new(&mut target); + inode.readlink(&mut target).await?; + + self.walk_recursive( + &symlink.parent().clone(), + Path::new(target.data()).unwrap(), + nr_follows + 1, + ) + .await + } + + fn follow_symlink_boxed<'r, 'a: 'r, 'b: 'r, 'c: 'r>( + &'a self, + symlink: ArcRef<'b, Dentry>, + inode: &'c InodeUse, + nr_follows: u32, + ) -> Pin<Box<dyn Future<Output = KResult<Arc<Dentry>>> + Send + 'r>> { + Box::pin(self.follow_symlink(symlink, inode, nr_follows)) + } + + async fn walk_recursive( + &self, + cwd: &Arc<Dentry>, + path: &Path, + nr_follows: u32, + ) -> KResult<Arc<Dentry>> { + const MAX_NR_FOLLOWS: u32 = 16; + + let mut current_owned; + let mut current; + if path.is_absolute() { + current = self.fsroot.aref(); + } else { + current = cwd.aref(); + } + + let mut path_iter = path.iter(); + + loop { + match self.walk_full(current, &mut path_iter).await { + WalkResult::Err(posix_error) => return Err(posix_error.into()), + WalkResult::Ok(dentry) => return Ok(dentry), + WalkResult::Symlink { symlink, inode } => { + if nr_follows >= MAX_NR_FOLLOWS { + return Err(ELOOP); + } + + current_owned = self + .follow_symlink_boxed(symlink.aref(), &inode, nr_follows) + .await?; + current = current_owned.aref(); + } + } + } + } + + pub async fn start_recursive_walk( + &self, + cwd: &Arc<Dentry>, + path: &Path, + ) -> KResult<Arc<Dentry>> { + self.walk_recursive(cwd, path, 0).await + } +} + +impl<'a, 'b> DentryFind<'a, 'b> { + fn new(parent: &'a Dentry, name: &'b [u8]) -> Self { + let builder: BuildHasherDefault<KernelHasher> = Default::default(); + let mut hasher = builder.build_hasher(); + + hasher.write_usize(parent as *const _ as usize); + hasher.write(name); + let hash = hasher.finish() as usize; + + Self { parent, name, hash } + } +} + +impl DCacheItem for DentryFind<'_, '_> { + fn d_hash(&self) -> usize { + self.hash + } + + fn d_parent(&self) -> *const Dentry { + self.parent as *const _ + } + + fn d_name<'r, 'a: 'r, 'b: 'a>( + &'a self, + _rcu_read: &'b RCUReadLock, + ) -> impl Deref<Target = [u8]> + 'r { + self.name + } +} diff --git a/src/kernel/vfs/file/inode_file.rs b/src/kernel/vfs/file/inode_file.rs index 6386ba92..d302079c 100644 --- a/src/kernel/vfs/file/inode_file.rs +++ b/src/kernel/vfs/file/inode_file.rs @@ -1,23 +1,17 @@ -use super::{File, FileType, SeekOption}; -use crate::{ - io::{Buffer, BufferFill, Stream}, - kernel::{ - constants::{EBADF, EFAULT, ENOTDIR, EOVERFLOW, ESPIPE}, - vfs::{ - dentry::Dentry, - inode::{Inode, Mode, WriteOffset}, - }, - }, - prelude::KResult, -}; use alloc::sync::Arc; -use core::{ops::ControlFlow, sync::atomic::Ordering}; + use eonix_sync::Mutex; -use posix_types::{ - getdent::{UserDirent, UserDirent64}, - open::OpenFlags, - stat::StatX, -}; +use posix_types::getdent::{UserDirent, UserDirent64}; +use posix_types::open::OpenFlags; +use posix_types::stat::StatX; + +use super::{File, FileType, SeekOption}; +use crate::io::{Buffer, BufferFill, Stream}; +use crate::kernel::constants::{EBADF, EFAULT, ENOTDIR, EOVERFLOW, ESPIPE}; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::inode::{InodeUse, WriteOffset}; +use crate::kernel::vfs::types::Format; +use crate::prelude::KResult; pub struct InodeFile { pub r: bool, @@ -25,7 +19,7 @@ pub struct InodeFile { pub a: bool, /// Only a few modes those won't possibly change are cached here to speed up file operations. /// Specifically, `S_IFMT` masked bits. - pub mode: Mode, + pub format: Format, cursor: Mutex<usize>, dentry: Arc<Dentry>, } @@ -34,12 +28,7 @@ impl InodeFile { pub fn new(dentry: Arc<Dentry>, flags: OpenFlags) -> File { // SAFETY: `dentry` used to create `InodeFile` is valid. // SAFETY: `mode` should never change with respect to the `S_IFMT` fields. - let cached_mode = dentry - .get_inode() - .expect("`dentry` is invalid") - .mode - .load() - .format(); + let format = dentry.inode().expect("dentry should be invalid").format; let (r, w, a) = flags.as_rwa(); @@ -50,15 +39,15 @@ impl InodeFile { r, w, a, - mode: cached_mode, + format, cursor: Mutex::new(0), }), ) } pub fn sendfile_check(&self) -> KResult<()> { - match self.mode { - Mode::REG | Mode::BLK => Ok(()), + match self.format { + Format::REG | Format::BLK => Ok(()), _ => Err(EBADF), } } @@ -70,21 +59,19 @@ impl InodeFile { let mut cursor = self.cursor.lock().await; - if self.a { - let nwrote = self.dentry.write(stream, WriteOffset::End(&mut cursor))?; + let (offset, update_offset) = match (self.a, offset) { + (true, _) => (WriteOffset::End(&mut cursor), None), + (false, Some(offset)) => (WriteOffset::Position(offset), None), + (false, None) => (WriteOffset::Position(*cursor), Some(&mut *cursor)), + }; - Ok(nwrote) - } else { - let nwrote = if let Some(offset) = offset { - self.dentry.write(stream, WriteOffset::Position(offset))? - } else { - let nwrote = self.dentry.write(stream, WriteOffset::Position(*cursor))?; - *cursor += nwrote; - nwrote - }; - - Ok(nwrote) + let nr_write = self.dentry.write(stream, offset).await?; + + if let Some(update_offset) = update_offset { + *update_offset += nr_write; } + + Ok(nr_write) } pub async fn read(&self, buffer: &mut dyn Buffer, offset: Option<usize>) -> KResult<usize> { @@ -92,24 +79,20 @@ impl InodeFile { return Err(EBADF); } - let nread = if let Some(offset) = offset { - let nread = self.dentry.read(buffer, offset)?; - nread - } else { - let mut cursor = self.cursor.lock().await; - - let nread = self.dentry.read(buffer, *cursor)?; + if let Some(offset) = offset { + return Ok(self.dentry.read(buffer, offset).await?); + } - *cursor += nread; - nread - }; + let mut cursor = self.cursor.lock().await; + let nread = self.dentry.read(buffer, *cursor).await?; + *cursor += nread; Ok(nread) } } impl File { - pub fn get_inode(&self) -> KResult<Option<Arc<dyn Inode>>> { + pub fn get_inode(&self) -> KResult<Option<InodeUse>> { if let FileType::Inode(inode_file) = &**self { Ok(Some(inode_file.dentry.get_inode()?)) } else { @@ -124,27 +107,30 @@ impl File { let mut cursor = inode_file.cursor.lock().await; - let nread = inode_file.dentry.readdir(*cursor, |filename, ino| { - // + 1 for filename length padding '\0', + 1 for d_type. - let real_record_len = core::mem::size_of::<UserDirent>() + filename.len() + 2; + let nread = inode_file + .dentry + .readdir(*cursor, |filename, ino| { + // + 1 for filename length padding '\0', + 1 for d_type. + let real_record_len = core::mem::size_of::<UserDirent>() + filename.len() + 2; - if buffer.available() < real_record_len { - return Ok(ControlFlow::Break(())); - } + if buffer.available() < real_record_len { + return Ok(false); + } - let record = UserDirent { - d_ino: ino as u32, - d_off: 0, - d_reclen: real_record_len as u16, - d_name: [0; 0], - }; + let record = UserDirent { + d_ino: ino.as_raw() as u32, + d_off: 0, + d_reclen: real_record_len as u16, + d_name: [0; 0], + }; - buffer.copy(&record)?.ok_or(EFAULT)?; - buffer.fill(filename)?.ok_or(EFAULT)?; - buffer.fill(&[0, 0])?.ok_or(EFAULT)?; + buffer.copy(&record)?.ok_or(EFAULT)?; + buffer.fill(filename)?.ok_or(EFAULT)?; + buffer.fill(&[0, 0])?.ok_or(EFAULT)?; - Ok(ControlFlow::Continue(())) - })?; + Ok(true) + }) + .await??; *cursor += nread; Ok(()) @@ -157,28 +143,31 @@ impl File { let mut cursor = inode_file.cursor.lock().await; - let nread = inode_file.dentry.readdir(*cursor, |filename, ino| { - // Filename length + 1 for padding '\0' - let real_record_len = core::mem::size_of::<UserDirent64>() + filename.len() + 1; + let nread = inode_file + .dentry + .readdir(*cursor, |filename, ino| { + // Filename length + 1 for padding '\0' + let real_record_len = core::mem::size_of::<UserDirent64>() + filename.len() + 1; - if buffer.available() < real_record_len { - return Ok(ControlFlow::Break(())); - } + if buffer.available() < real_record_len { + return Ok(false); + } - let record = UserDirent64 { - d_ino: ino, - d_off: 0, - d_reclen: real_record_len as u16, - d_type: 0, - d_name: [0; 0], - }; + let record = UserDirent64 { + d_ino: ino.as_raw(), + d_off: 0, + d_reclen: real_record_len as u16, + d_type: 0, + d_name: [0; 0], + }; - buffer.copy(&record)?.ok_or(EFAULT)?; - buffer.fill(filename)?.ok_or(EFAULT)?; - buffer.fill(&[0])?.ok_or(EFAULT)?; + buffer.copy(&record)?.ok_or(EFAULT)?; + buffer.fill(filename)?.ok_or(EFAULT)?; + buffer.fill(&[0])?.ok_or(EFAULT)?; - Ok(ControlFlow::Continue(())) - })?; + Ok(true) + }) + .await??; *cursor += nread; Ok(()) @@ -196,7 +185,7 @@ impl File { SeekOption::Set(n) => n, SeekOption::End(off) => { let inode = inode_file.dentry.get_inode()?; - let size = inode.size.load(Ordering::Relaxed) as usize; + let size = inode.info.lock().size as usize; size.checked_add_signed(off).ok_or(EOVERFLOW)? } }; diff --git a/src/kernel/vfs/file/mod.rs b/src/kernel/vfs/file/mod.rs index bb1c66ec..0af79dc8 100644 --- a/src/kernel/vfs/file/mod.rs +++ b/src/kernel/vfs/file/mod.rs @@ -2,29 +2,24 @@ mod inode_file; mod pipe; mod terminal_file; -use crate::{ - io::{Buffer, ByteBuffer, Chunks, IntoStream, Stream}, - kernel::{ - constants::{EBADF, EINTR, EINVAL, ENOTTY}, - mem::{AsMemoryBlock, Page}, - task::Thread, - CharDevice, - }, - prelude::KResult, -}; use alloc::sync::Arc; -use bitflags::bitflags; -use core::{ - ops::Deref, - sync::atomic::{AtomicI32, AtomicU32, Ordering}, -}; -use pipe::{PipeReadEnd, PipeWriteEnd}; -use posix_types::open::OpenFlags; +use core::ops::Deref; +use core::sync::atomic::{AtomicI32, AtomicU32, Ordering}; +use bitflags::bitflags; pub use inode_file::InodeFile; pub use pipe::Pipe; +use pipe::{PipeReadEnd, PipeWriteEnd}; +use posix_types::open::OpenFlags; pub use terminal_file::TerminalFile; +use crate::io::{Buffer, ByteBuffer, Chunks, IntoStream, Stream}; +use crate::kernel::constants::{EBADF, EINTR, EINVAL, ENOTTY}; +use crate::kernel::mem::FolioOwned; +use crate::kernel::task::Thread; +use crate::kernel::CharDevice; +use crate::prelude::KResult; + pub enum FileType { Inode(InodeFile), PipeRead(PipeReadEnd), @@ -56,12 +51,14 @@ bitflags! { } impl FileType { - pub async fn read(&self, buffer: &mut dyn Buffer, offset: Option<usize>) -> KResult<usize> { + pub async fn read( + &self, buffer: &mut dyn Buffer, offset: Option<usize>, + ) -> KResult<usize> { match self { FileType::Inode(inode) => inode.read(buffer, offset).await, FileType::PipeRead(pipe) => pipe.read(buffer).await, FileType::Terminal(tty) => tty.read(buffer).await, - FileType::CharDev(device) => device.read(buffer), + FileType::CharDev(device) => device.read(buffer).await, _ => Err(EBADF), } } @@ -81,7 +78,9 @@ impl FileType { // } // } - pub async fn write(&self, stream: &mut dyn Stream, offset: Option<usize>) -> KResult<usize> { + pub async fn write( + &self, stream: &mut dyn Stream, offset: Option<usize>, + ) -> KResult<usize> { match self { FileType::Inode(inode) => inode.write(stream, offset).await, FileType::PipeWrite(pipe) => pipe.write(stream).await, @@ -98,10 +97,11 @@ impl FileType { } } - pub async fn sendfile(&self, dest_file: &Self, count: usize) -> KResult<usize> { - let buffer_page = Page::alloc(); - // SAFETY: We are the only owner of the page. - let buffer = unsafe { buffer_page.as_memblk().as_bytes_mut() }; + pub async fn sendfile( + &self, dest_file: &Self, count: usize, + ) -> KResult<usize> { + let mut buffer_page = FolioOwned::alloc(); + let buffer = buffer_page.as_bytes_mut(); self.sendfile_check()?; @@ -132,7 +132,9 @@ impl FileType { pub async fn ioctl(&self, request: usize, arg3: usize) -> KResult<usize> { match self { - FileType::Terminal(tty) => tty.ioctl(request, arg3).await.map(|_| 0), + FileType::Terminal(tty) => { + tty.ioctl(request, arg3).await.map(|_| 0) + } _ => Err(ENOTTY), } } diff --git a/src/kernel/vfs/file/terminal_file.rs b/src/kernel/vfs/file/terminal_file.rs index f318c5b2..04a022b5 100644 --- a/src/kernel/vfs/file/terminal_file.rs +++ b/src/kernel/vfs/file/terminal_file.rs @@ -1,24 +1,46 @@ -use super::{File, FileType, PollEvent}; -use crate::{ - io::{Buffer, Stream, StreamRead}, - kernel::{ - constants::{EINVAL, TCGETS, TCSETS, TIOCGPGRP, TIOCGWINSZ, TIOCSPGRP}, - terminal::TerminalIORequest, - user::{UserPointer, UserPointerMut}, - Terminal, - }, - prelude::KResult, -}; use alloc::sync::Arc; + +use eonix_sync::AsProof; use posix_types::open::OpenFlags; +use super::{File, FileType, PollEvent}; +use crate::io::{Buffer, Stream, StreamRead}; +use crate::kernel::constants::{ + EINVAL, TCGETS, TCSETS, TIOCGPGRP, TIOCGWINSZ, TIOCSPGRP, +}; +use crate::kernel::task::{ProcessList, Thread}; +use crate::kernel::terminal::TerminalIORequest; +use crate::kernel::user::{UserPointer, UserPointerMut}; +use crate::kernel::Terminal; +use crate::prelude::KResult; + pub struct TerminalFile { terminal: Arc<Terminal>, } impl TerminalFile { - pub fn new(tty: Arc<Terminal>, flags: OpenFlags) -> File { - File::new(flags, FileType::Terminal(TerminalFile { terminal: tty })) + pub async fn open( + thread: &Thread, + terminal: &Arc<Terminal>, + flags: OpenFlags, + ) -> File { + let set_control_tty = !flags.contains(OpenFlags::O_NOCTTY); + + let procs = ProcessList::get().read().await; + let session = thread.process.session(procs.prove()); + + // We only set the control terminal if the process is the session leader. + if set_control_tty && session.sid == thread.process.pid { + // Silently fail if we can't set the control terminal. + let _ = terminal.set_session(&session, false, procs.prove()).await; + } + + File::new( + flags, + FileType::Terminal(TerminalFile { + terminal: terminal.clone(), + }), + ) } pub async fn read(&self, buffer: &mut dyn Buffer) -> KResult<usize> { @@ -43,11 +65,21 @@ impl TerminalFile { pub async fn ioctl(&self, request: usize, arg3: usize) -> KResult<()> { self.terminal .ioctl(match request as u32 { - TCGETS => TerminalIORequest::GetTermios(UserPointerMut::with_addr(arg3)?), - TCSETS => TerminalIORequest::SetTermios(UserPointer::with_addr(arg3)?), - TIOCGPGRP => TerminalIORequest::GetProcessGroup(UserPointerMut::with_addr(arg3)?), - TIOCSPGRP => TerminalIORequest::SetProcessGroup(UserPointer::with_addr(arg3)?), - TIOCGWINSZ => TerminalIORequest::GetWindowSize(UserPointerMut::with_addr(arg3)?), + TCGETS => TerminalIORequest::GetTermios( + UserPointerMut::with_addr(arg3)?, + ), + TCSETS => { + TerminalIORequest::SetTermios(UserPointer::with_addr(arg3)?) + } + TIOCGPGRP => TerminalIORequest::GetProcessGroup( + UserPointerMut::with_addr(arg3)?, + ), + TIOCSPGRP => TerminalIORequest::SetProcessGroup( + UserPointer::with_addr(arg3)?, + ), + TIOCGWINSZ => TerminalIORequest::GetWindowSize( + UserPointerMut::with_addr(arg3)?, + ), _ => return Err(EINVAL), }) .await diff --git a/src/kernel/vfs/filearray.rs b/src/kernel/vfs/filearray.rs index b457a425..c0b6a49e 100644 --- a/src/kernel/vfs/filearray.rs +++ b/src/kernel/vfs/filearray.rs @@ -1,28 +1,26 @@ -use super::{ - file::{File, InodeFile, Pipe}, - inode::Mode, - Spin, TerminalFile, -}; -use crate::kernel::{ - constants::{ - EBADF, EISDIR, ENOTDIR, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, F_SETFD, F_SETFL, - }, - syscall::{FromSyscallArg, SyscallRetVal}, -}; -use crate::{ - kernel::{console::get_console, constants::ENXIO, vfs::dentry::Dentry, CharDevice}, - prelude::*, -}; use alloc::sync::Arc; + +use intrusive_collections::rbtree::Entry; use intrusive_collections::{ - intrusive_adapter, rbtree::Entry, Bound, KeyAdapter, RBTree, RBTreeAtomicLink, -}; -use itertools::{ - FoldWhile::{Continue, Done}, - Itertools, + intrusive_adapter, Bound, KeyAdapter, RBTree, RBTreeAtomicLink, }; +use itertools::FoldWhile::{Continue, Done}; +use itertools::Itertools; use posix_types::open::{FDFlags, OpenFlags}; +use super::file::{File, InodeFile, Pipe}; +use super::types::{Format, Permission}; +use super::Spin; +use crate::kernel::constants::{ + EBADF, EISDIR, ENOTDIR, ENXIO, F_DUPFD, F_DUPFD_CLOEXEC, F_GETFD, F_GETFL, + F_SETFD, F_SETFL, +}; +use crate::kernel::syscall::{FromSyscallArg, SyscallRetVal}; +use crate::kernel::task::Thread; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::CharDevice; +use crate::prelude::*; + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct FD(u32); @@ -85,7 +83,11 @@ impl FDAllocator { self.min_avail = FD(0); } - fn find_available(&mut self, from: FD, files: &RBTree<OpenFileAdapter>) -> FD { + fn find_available( + &mut self, + from: FD, + files: &RBTree<OpenFileAdapter>, + ) -> FD { files .range(Bound::Included(&from), Bound::Unbounded) .fold_while(from, |current, OpenFile { fd, .. }| { @@ -148,7 +150,8 @@ impl FileArray { let other_inner = other.inner.lock(); for file in other_inner.files.iter() { - let new_file = OpenFile::new(file.fd, file.flags, file.file.dup()); + let new_file = + OpenFile::new(file.fd, file.flags, file.file.dup()); new_files.insert(new_file); } (new_files, other_inner.fd_alloc.clone()) @@ -228,7 +231,12 @@ impl FileArray { /// Duplicates the file to a new file descriptor, returning the old file /// description to be dropped. - fn dup_to_no_close(&self, old_fd: FD, new_fd: FD, fd_flags: FDFlags) -> KResult<Option<File>> { + fn dup_to_no_close( + &self, + old_fd: FD, + new_fd: FD, + fd_flags: FDFlags, + ) -> KResult<Option<File>> { let mut inner = self.inner.lock(); let (files, fd_alloc) = inner.split_borrow(); @@ -245,7 +253,8 @@ impl FileArray { Entry::Occupied(mut entry) => { let mut file = entry.remove().unwrap(); file.flags = fd_flags; - let old_file = core::mem::replace(&mut file.file, new_file_data); + let old_file = + core::mem::replace(&mut file.file, new_file_data); entry.insert(file); @@ -254,8 +263,15 @@ impl FileArray { } } - pub async fn dup_to(&self, old_fd: FD, new_fd: FD, flags: OpenFlags) -> KResult<FD> { - if let Some(old_file) = self.dup_to_no_close(old_fd, new_fd, flags.as_fd_flags())? { + pub async fn dup_to( + &self, + old_fd: FD, + new_fd: FD, + flags: OpenFlags, + ) -> KResult<FD> { + if let Some(old_file) = + self.dup_to_no_close(old_fd, new_fd, flags.as_fd_flags())? + { old_file.close().await; } @@ -280,28 +296,33 @@ impl FileArray { Ok((read_fd, write_fd)) } - pub fn open(&self, dentry: &Arc<Dentry>, flags: OpenFlags, mode: Mode) -> KResult<FD> { - dentry.open_check(flags, mode)?; + pub async fn open( + &self, + thread: &Thread, + dentry: &Arc<Dentry>, + flags: OpenFlags, + perm: Permission, + ) -> KResult<FD> { + dentry.open_check(flags, perm).await?; let fdflag = flags.as_fd_flags(); let inode = dentry.get_inode()?; - let file_format = inode.mode.load().format(); - match (flags.directory(), file_format, flags.write()) { - (true, Mode::DIR, _) => {} + match (flags.directory(), inode.format, flags.write()) { + (true, Format::DIR, _) => {} (true, _, _) => return Err(ENOTDIR), - (false, Mode::DIR, true) => return Err(EISDIR), + (false, Format::DIR, true) => return Err(EISDIR), _ => {} } - if flags.truncate() && flags.write() && file_format.is_reg() { - inode.truncate(0)?; + if flags.truncate() && flags.write() && inode.format == Format::REG { + inode.truncate(0).await?; } - let file = if file_format.is_chr() { + let file = if inode.format == Format::CHR { let device = CharDevice::get(inode.devid()?).ok_or(ENXIO)?; - device.open(flags)? + device.open(thread, flags).await? } else { InodeFile::new(dentry.clone(), flags) }; @@ -324,7 +345,8 @@ impl FileArray { F_DUPFD | F_DUPFD_CLOEXEC => { let ofile = cursor.get().ok_or(EBADF)?; - let cloexec = cmd == F_DUPFD_CLOEXEC || ofile.flags.close_on_exec(); + let cloexec = + cmd == F_DUPFD_CLOEXEC || ofile.flags.close_on_exec(); let flags = cloexec .then_some(FDFlags::FD_CLOEXEC) .unwrap_or(FDFlags::empty()); @@ -343,7 +365,9 @@ impl FileArray { cursor.insert(ofile); 0 } - F_GETFL => cursor.get().ok_or(EBADF)?.file.get_flags().bits() as usize, + F_GETFL => { + cursor.get().ok_or(EBADF)?.file.get_flags().bits() as usize + } F_SETFL => { cursor .get() @@ -358,35 +382,6 @@ impl FileArray { Ok(ret) } - - /// Only used for init process. - pub fn open_console(&self) { - let mut inner = self.inner.lock(); - let (files, fd_alloc) = inner.split_borrow(); - - let (stdin, stdout, stderr) = ( - fd_alloc.next_fd(files), - fd_alloc.next_fd(files), - fd_alloc.next_fd(files), - ); - let console_terminal = get_console().expect("No console terminal"); - - inner.do_insert( - stdin, - FDFlags::FD_CLOEXEC, - TerminalFile::new(console_terminal.clone(), OpenFlags::empty()), - ); - inner.do_insert( - stdout, - FDFlags::FD_CLOEXEC, - TerminalFile::new(console_terminal.clone(), OpenFlags::empty()), - ); - inner.do_insert( - stderr, - FDFlags::FD_CLOEXEC, - TerminalFile::new(console_terminal.clone(), OpenFlags::empty()), - ); - } } impl FileArrayInner { @@ -398,7 +393,9 @@ impl FileArrayInner { fn do_insert(&mut self, fd: FD, flags: FDFlags, file: File) { match self.files.entry(&fd) { Entry::Occupied(_) => { - panic!("File descriptor {fd:?} already exists in the file array."); + panic!( + "File descriptor {fd:?} already exists in the file array." + ); } Entry::Vacant(insert_cursor) => { insert_cursor.insert(OpenFile::new(fd, flags, file)); @@ -406,7 +403,9 @@ impl FileArrayInner { } } - fn split_borrow(&mut self) -> (&mut RBTree<OpenFileAdapter>, &mut FDAllocator) { + fn split_borrow( + &mut self, + ) -> (&mut RBTree<OpenFileAdapter>, &mut FDAllocator) { let Self { files, fd_alloc } = self; (files, fd_alloc) } diff --git a/src/kernel/vfs/inode.rs b/src/kernel/vfs/inode.rs deleted file mode 100644 index 52529f84..00000000 --- a/src/kernel/vfs/inode.rs +++ /dev/null @@ -1,494 +0,0 @@ -use super::{dentry::Dentry, vfs::Vfs, DevId}; -use crate::io::Stream; -use crate::kernel::constants::{ - EINVAL, EISDIR, ENOTDIR, EPERM, STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, - STATX_MODE, STATX_MTIME, STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, S_IFBLK, S_IFCHR, - S_IFDIR, S_IFLNK, S_IFMT, S_IFREG, -}; -use crate::kernel::mem::PageCache; -use crate::kernel::syscall::{FromSyscallArg, SyscallRetVal}; -use crate::kernel::task::block_on; -use crate::kernel::timer::Instant; -use crate::{io::Buffer, prelude::*}; -use alloc::sync::{Arc, Weak}; -use core::{ - mem::MaybeUninit, - ops::ControlFlow, - ptr::addr_of_mut, - sync::atomic::{AtomicU32, AtomicU64, Ordering}, -}; -use eonix_sync::RwLock; -use posix_types::stat::StatX; - -pub type Ino = u64; -pub type AtomicIno = AtomicU64; -#[allow(dead_code)] -pub type ISize = u64; -pub type AtomicISize = AtomicU64; -#[allow(dead_code)] -pub type Nlink = u64; -pub type AtomicNlink = AtomicU64; -#[allow(dead_code)] -pub type Uid = u32; -pub type AtomicUid = AtomicU32; -#[allow(dead_code)] -pub type Gid = u32; -pub type AtomicGid = AtomicU32; - -#[derive(Clone, Copy, PartialEq, Eq)] -pub struct Mode(u32); - -pub struct AtomicMode(AtomicU32); - -#[derive(Debug)] -pub struct InodeData { - pub ino: Ino, - pub size: AtomicISize, - pub nlink: AtomicNlink, - - pub uid: AtomicUid, - pub gid: AtomicGid, - pub mode: AtomicMode, - - pub atime: Spin<Instant>, - pub ctime: Spin<Instant>, - pub mtime: Spin<Instant>, - - pub rwsem: RwLock<()>, - - pub vfs: Weak<dyn Vfs>, -} - -impl InodeData { - pub fn new(ino: Ino, vfs: Weak<dyn Vfs>) -> Self { - Self { - ino, - vfs, - atime: Spin::new(Instant::now()), - ctime: Spin::new(Instant::now()), - mtime: Spin::new(Instant::now()), - rwsem: RwLock::new(()), - size: AtomicU64::new(0), - nlink: AtomicNlink::new(0), - uid: AtomicUid::new(0), - gid: AtomicGid::new(0), - mode: AtomicMode::new(0), - } - } -} - -#[allow(dead_code)] -pub trait InodeInner: - Send + Sync + core::ops::Deref<Target = InodeData> + core::ops::DerefMut -{ - fn data(&self) -> &InodeData; - fn data_mut(&mut self) -> &mut InodeData; -} - -pub enum WriteOffset<'end> { - Position(usize), - End(&'end mut usize), -} - -pub struct RenameData<'a, 'b> { - pub old_dentry: &'a Arc<Dentry>, - pub new_dentry: &'b Arc<Dentry>, - pub new_parent: Arc<dyn Inode>, - pub vfs: Arc<dyn Vfs>, - pub is_exchange: bool, - pub no_replace: bool, -} - -#[allow(unused_variables)] -pub trait Inode: Send + Sync + InodeInner + Any { - fn is_dir(&self) -> bool { - self.mode.load().is_dir() - } - - fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<Arc<dyn Inode>>> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn creat(&self, at: &Arc<Dentry>, mode: Mode) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn mkdir(&self, at: &Dentry, mode: Mode) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn mknod(&self, at: &Dentry, mode: Mode, dev: DevId) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn write(&self, stream: &mut dyn Stream, offset: WriteOffset) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn write_direct(&self, stream: &mut dyn Stream, offset: usize) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn devid(&self) -> KResult<DevId> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { - Err(if self.is_dir() { EISDIR } else { EINVAL }) - } - - fn truncate(&self, length: usize) -> KResult<()> { - Err(if self.is_dir() { EISDIR } else { EPERM }) - } - - fn rename(&self, rename_data: RenameData) -> KResult<()> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn do_readdir( - &self, - offset: usize, - callback: &mut dyn FnMut(&[u8], Ino) -> KResult<ControlFlow<(), ()>>, - ) -> KResult<usize> { - Err(if !self.is_dir() { ENOTDIR } else { EPERM }) - } - - fn chmod(&self, mode: Mode) -> KResult<()> { - Err(EPERM) - } - - fn chown(&self, uid: u32, gid: u32) -> KResult<()> { - Err(EPERM) - } - - fn page_cache(&self) -> Option<&PageCache> { - None - } - - fn statx(&self, stat: &mut StatX, mask: u32) -> KResult<()> { - // Safety: ffi should have checked reference - let vfs = self.vfs.upgrade().expect("Vfs is dropped"); - - let size = self.size.load(Ordering::Relaxed); - let mode = self.mode.load(); - - if mask & STATX_NLINK != 0 { - stat.stx_nlink = self.nlink.load(Ordering::Acquire) as _; - stat.stx_mask |= STATX_NLINK; - } - - if mask & STATX_ATIME != 0 { - let atime = *self.atime.lock(); - stat.stx_atime = atime.into(); - stat.stx_mask |= STATX_ATIME; - } - - if mask & STATX_MTIME != 0 { - let mtime = *self.mtime.lock(); - stat.stx_mtime = mtime.into(); - stat.stx_mask |= STATX_MTIME; - } - - if mask & STATX_CTIME != 0 { - let ctime = *self.ctime.lock(); - stat.stx_ctime = ctime.into(); - stat.stx_mask |= STATX_CTIME; - } - - if mask & STATX_SIZE != 0 { - stat.stx_size = self.size.load(Ordering::Relaxed) as _; - stat.stx_mask |= STATX_SIZE; - } - - stat.stx_mode = 0; - if mask & STATX_MODE != 0 { - stat.stx_mode |= mode.non_format_bits() as u16; - stat.stx_mask |= STATX_MODE; - } - - if mask & STATX_TYPE != 0 { - stat.stx_mode |= mode.format_bits() as u16; - if mode.is_blk() || mode.is_chr() { - let devid = self.devid(); - stat.stx_rdev_major = (devid? >> 8) & 0xff; - stat.stx_rdev_minor = devid? & 0xff; - } - stat.stx_mask |= STATX_TYPE; - } - - if mask & STATX_INO != 0 { - stat.stx_ino = self.ino as _; - stat.stx_mask |= STATX_INO; - } - - if mask & STATX_BLOCKS != 0 { - stat.stx_blocks = (size + 512 - 1) / 512; - stat.stx_blksize = vfs.io_blksize() as _; - stat.stx_mask |= STATX_BLOCKS; - } - - if mask & STATX_UID != 0 { - stat.stx_uid = self.uid.load(Ordering::Relaxed) as _; - stat.stx_mask |= STATX_UID; - } - - if mask & STATX_GID != 0 { - stat.stx_gid = self.gid.load(Ordering::Relaxed) as _; - stat.stx_mask |= STATX_GID; - } - - let fsdev = vfs.fs_devid(); - stat.stx_dev_major = (fsdev >> 8) & 0xff; - stat.stx_dev_minor = fsdev & 0xff; - - // TODO: support more attributes - stat.stx_attributes_mask = 0; - - Ok(()) - } - - fn new_locked<F>(ino: Ino, vfs: Weak<dyn Vfs>, f: F) -> Arc<Self> - where - Self: Sized, - F: FnOnce(*mut Self, &()), - { - let mut uninit = Arc::<Self>::new_uninit(); - - let uninit_mut = Arc::get_mut(&mut uninit).unwrap(); - - // Safety: `idata` is owned by `uninit` - let idata = unsafe { - addr_of_mut!(*(*uninit_mut.as_mut_ptr()).data_mut()) - .cast::<MaybeUninit<InodeData>>() - .as_mut() - .unwrap() - }; - - idata.write(InodeData::new(ino, vfs)); - - f( - uninit_mut.as_mut_ptr(), - // SAFETY: `idata` is initialized and we will never move the lock. - &block_on(unsafe { idata.assume_init_ref() }.rwsem.read()), - ); - - // Safety: `uninit` is initialized - unsafe { uninit.assume_init() } - } -} - -// TODO: define multiple inode structs a time -macro_rules! define_struct_inode { - ($v:vis struct $inode_t:ident;) => { - $v struct $inode_t { - /// Do not use this directly - idata: $crate::kernel::vfs::inode::InodeData, - } - - impl core::ops::Deref for $inode_t { - type Target = $crate::kernel::vfs::inode::InodeData; - - fn deref(&self) -> &Self::Target { - &self.idata - } - } - - impl core::ops::DerefMut for $inode_t { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.idata - } - } - - impl $crate::kernel::vfs::inode::InodeInner for $inode_t { - fn data(&self) -> &$crate::kernel::vfs::inode::InodeData { - &self.idata - } - - fn data_mut(&mut self) -> &mut $crate::kernel::vfs::inode::InodeData { - &mut self.idata - } - } - }; - ($v:vis struct $inode_t:ident { $($vis:vis $name:ident: $type:ty,)* }) => { - $v struct $inode_t { - /// Do not use this directly - idata: $crate::kernel::vfs::inode::InodeData, - $($vis $name: $type,)* - } - - impl core::ops::Deref for $inode_t { - type Target = $crate::kernel::vfs::inode::InodeData; - - fn deref(&self) -> &Self::Target { - &self.idata - } - } - - impl core::ops::DerefMut for $inode_t { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.idata - } - } - - impl $crate::kernel::vfs::inode::InodeInner for $inode_t { - fn data(&self) -> &$crate::kernel::vfs::inode::InodeData { - &self.idata - } - - fn data_mut(&mut self) -> &mut $crate::kernel::vfs::inode::InodeData { - &mut self.idata - } - } - }; -} - -pub(crate) use define_struct_inode; - -impl Mode { - pub const REG: Self = Self(S_IFREG); - pub const DIR: Self = Self(S_IFDIR); - pub const LNK: Self = Self(S_IFLNK); - pub const BLK: Self = Self(S_IFBLK); - pub const CHR: Self = Self(S_IFCHR); - - pub const fn new(bits: u32) -> Self { - Self(bits) - } - - pub const fn is_blk(&self) -> bool { - (self.0 & S_IFMT) == S_IFBLK - } - - pub const fn is_chr(&self) -> bool { - (self.0 & S_IFMT) == S_IFCHR - } - - pub const fn is_reg(&self) -> bool { - (self.0 & S_IFMT) == S_IFREG - } - - pub const fn is_dir(&self) -> bool { - (self.0 & S_IFMT) == S_IFDIR - } - - pub const fn is_lnk(&self) -> bool { - (self.0 & S_IFMT) == S_IFLNK - } - - pub const fn bits(&self) -> u32 { - self.0 - } - - pub const fn format_bits(&self) -> u32 { - self.0 & S_IFMT - } - - pub const fn format(&self) -> Self { - Self::new(self.format_bits()) - } - - pub const fn non_format_bits(&self) -> u32 { - self.0 & !S_IFMT - } - - pub const fn non_format(&self) -> Self { - Self::new(self.non_format_bits()) - } - - pub const fn perm(self, perm: u32) -> Self { - Self::new((self.0 & !0o777) | (perm & 0o777)) - } - - pub const fn set_perm(&mut self, perm: u32) { - *self = self.perm(perm); - } - - pub const fn mask_perm(&mut self, perm_mask: u32) { - let perm_mask = perm_mask & 0o777; - let self_perm = self.non_format_bits() & 0o777; - - *self = self.perm(self_perm & perm_mask); - } -} - -impl AtomicMode { - pub const fn new(bits: u32) -> Self { - Self(AtomicU32::new(bits)) - } - - pub const fn from(mode: Mode) -> Self { - Self::new(mode.0) - } - - pub fn load(&self) -> Mode { - Mode(self.0.load(Ordering::Relaxed)) - } - - pub fn store(&self, mode: Mode) { - self.0.store(mode.0, Ordering::Relaxed); - } -} - -impl core::fmt::Debug for AtomicMode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("AtomicMode") - .field("bits", &self.load().0) - .finish() - } -} - -impl core::fmt::Debug for Mode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let format_name = match self.format() { - Mode::REG => "REG", - Mode::DIR => "DIR", - Mode::LNK => "LNK", - Mode::BLK => "BLK", - Mode::CHR => "CHR", - _ => "UNK", - }; - - match self.non_format_bits() & !0o777 { - 0 => write!( - f, - "Mode({format_name}, {perm:#o})", - perm = self.non_format_bits() - )?, - rem => write!( - f, - "Mode({format_name}, {perm:#o}, rem={rem:#x})", - perm = self.non_format_bits() & 0o777 - )?, - } - - Ok(()) - } -} - -impl FromSyscallArg for Mode { - fn from_arg(value: usize) -> Self { - Mode::new(value as u32) - } -} - -impl SyscallRetVal for Mode { - fn into_retval(self) -> Option<usize> { - Some(self.bits() as usize) - } -} diff --git a/src/kernel/vfs/inode/ino.rs b/src/kernel/vfs/inode/ino.rs new file mode 100644 index 00000000..b5ee7ac0 --- /dev/null +++ b/src/kernel/vfs/inode/ino.rs @@ -0,0 +1,31 @@ +use core::{ + fmt::{Debug, Display, Formatter}, + sync::atomic::AtomicU64, +}; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Ino(u64); + +pub struct AtomicIno(AtomicU64); + +impl Ino { + pub const fn new(ino: u64) -> Self { + Self(ino) + } + + pub const fn as_raw(self) -> u64 { + self.0 + } +} + +impl Debug for Ino { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "Ino({})", self.0) + } +} + +impl Display for Ino { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/src/kernel/vfs/inode/inode.rs b/src/kernel/vfs/inode/inode.rs new file mode 100644 index 00000000..5f0b98c2 --- /dev/null +++ b/src/kernel/vfs/inode/inode.rs @@ -0,0 +1,363 @@ +use alloc::boxed::Box; +use alloc::collections::btree_map::BTreeMap; +use alloc::sync::{Arc, Weak}; +use core::any::Any; +use core::future::Future; +use core::ops::Deref; + +use async_trait::async_trait; +use eonix_sync::{RwLock, Spin}; + +use super::{Ino, RenameData, WriteOffset}; +use crate::io::{Buffer, Stream}; +use crate::kernel::constants::{EINVAL, EPERM}; +use crate::kernel::mem::{CachePage, PageCache, PageOffset}; +use crate::kernel::timer::Instant; +use crate::kernel::vfs::dentry::Dentry; +use crate::kernel::vfs::types::{DeviceId, Format, Mode, Permission}; +use crate::kernel::vfs::{SbRef, SbUse, SuperBlock}; +use crate::prelude::KResult; + +pub struct Inode { + pub ino: Ino, + pub format: Format, + pub info: Spin<InodeInfo>, + pub rwsem: RwLock<()>, + page_cache: Spin<Weak<PageCache>>, + sb: SbRef<dyn SuperBlock>, + ops: Box<dyn InodeOpsErased>, +} + +macro_rules! return_type { + ($type:ty) => { + $type + }; + () => { + () + }; +} + +macro_rules! define_inode_ops { + { + $( + $(#[$attr:meta])* + async fn $method:ident $(<$($lt:lifetime),+>)? (&self $(,)? $($name:ident : $type:ty $(,)?)*) $(-> $ret:ty)? + $body:block + )* + + --- + + $( + $(#[$attr1:meta])* + fn $method1:ident $(<$($lt1:lifetime),+>)? (&self $(,)? $($name1:ident : $type1:ty $(,)?)*) $(-> $ret1:ty)? + $body1:block + )* + } => { + #[allow(unused_variables)] + pub trait InodeOps: Sized + Send + Sync + 'static { + type SuperBlock: SuperBlock + Sized; + + $( + $(#[$attr])* + fn $method $(<$($lt),+>)? ( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + $($name : $type),* + ) -> impl Future<Output = return_type!($($ret)?)> + Send { + async { $body } + })* + + $( + $(#[$attr1])* + fn $method1 $(<$($lt1),+>)? ( + &self, + sb: SbUse<Self::SuperBlock>, + inode: &InodeUse, + $($name1 : $type1),* + ) -> return_type!($($ret1)?) { + $body1 + })* + } + + #[async_trait] + trait InodeOpsErased: Any + Send + Sync + 'static { + $(async fn $method $(<$($lt),+>)? ( + &self, + sb: SbUse<dyn SuperBlock>, + inode: &InodeUse, + $($name : $type),* + ) -> return_type!($($ret)?);)* + + $(fn $method1 $(<$($lt1),+>)? ( + &self, + sb: SbUse<dyn SuperBlock>, + inode: &InodeUse, + $($name1 : $type1),* + ) -> return_type!($($ret1)?);)* + } + + #[async_trait] + impl<T> InodeOpsErased for T + where + T: InodeOps, + { + $(async fn $method $(<$($lt),+>)? ( + &self, + sb: SbUse<dyn SuperBlock>, + inode: &InodeUse, + $($name : $type),* + ) -> return_type!($($ret)?) { + self.$method(sb.downcast(), inode, $($name),*).await + })* + + $(fn $method1 $(<$($lt1),+>)? ( + &self, + sb: SbUse<dyn SuperBlock>, + inode: &InodeUse, + $($name1 : $type1),* + ) -> return_type!($($ret1)?) { + self.$method1(sb.downcast(), inode, $($name1),*) + })* + } + + impl InodeUse { + $(pub async fn $method $(<$($lt),+>)? ( + &self, + $($name : $type),* + ) -> return_type!($($ret)?) { + self.ops.$method(self.sbget()?, self, $($name),*).await + })* + + $(pub fn $method1 $(<$($lt1),+>)? ( + &self, + $($name1 : $type1),* + ) -> return_type!($($ret1)?) { + self.ops.$method1(self.sbget()?, self, $($name1),*) + })* + } + }; +} + +define_inode_ops! { + // DIRECTORY OPERATIONS + + async fn lookup(&self, dentry: &Arc<Dentry>) -> KResult<Option<InodeUse>> { + Err(EPERM) + } + + /// Read directory entries and call the given closure for each entry. + /// + /// # Returns + /// - Ok(count): The number of entries read. + /// - Ok(Err(err)): Some error occurred while calling the given closure. + /// - Err(err): An error occurred while reading the directory. + async fn readdir( + &self, + offset: usize, + for_each_entry: &mut (dyn (for<'a> FnMut(&'a [u8], Ino) -> KResult<bool>) + Send), + ) -> KResult<KResult<usize>> { + Err(EPERM) + } + + async fn create(&self, at: &Arc<Dentry>, mode: Permission) -> KResult<()> { + Err(EPERM) + } + + async fn mkdir(&self, at: &Dentry, mode: Permission) -> KResult<()> { + Err(EPERM) + } + + async fn mknod(&self, at: &Dentry, mode: Mode, dev: DeviceId) -> KResult<()> { + Err(EPERM) + } + + async fn unlink(&self, at: &Arc<Dentry>) -> KResult<()> { + Err(EPERM) + } + + async fn symlink(&self, at: &Arc<Dentry>, target: &[u8]) -> KResult<()> { + Err(EPERM) + } + + async fn rename(&self, rename_data: RenameData<'_, '_>) -> KResult<()> { + Err(EPERM) + } + + // FILE OPERATIONS + + async fn read(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + Err(EINVAL) + } + + async fn read_direct(&self, buffer: &mut dyn Buffer, offset: usize) -> KResult<usize> { + Err(EINVAL) + } + + async fn write( + &self, + stream: &mut dyn Stream, + offset: WriteOffset<'_> + ) -> KResult<usize> { + Err(EINVAL) + } + + async fn write_direct( + &self, + stream: &mut dyn Stream, + offset: usize, + ) -> KResult<usize> { + Err(EINVAL) + } + + async fn readlink(&self, buffer: &mut dyn Buffer) -> KResult<usize> { + Err(EINVAL) + } + + async fn truncate(&self, length: usize) -> KResult<()> { + Err(EPERM) + } + + async fn chmod(&self, perm: Permission) -> KResult<()> { + Err(EPERM) + } + + async fn chown(&self, uid: u32, gid: u32) -> KResult<()> { + Err(EPERM) + } + + // PAGE CACHE OPERATIONS + async fn read_page(&self, page: &mut CachePage, offset: PageOffset) -> KResult<()> { + Err(EINVAL) + } + + async fn write_page(&self, page: &mut CachePage, offset: PageOffset) -> KResult<()> { + Err(EINVAL) + } + + async fn write_begin<'a>( + &self, + page_cache: &PageCache, + pages: &'a mut BTreeMap<PageOffset, CachePage>, + offset: usize, + len: usize, + ) -> KResult<&'a mut CachePage> { + Err(EINVAL) + } + + async fn write_end( + &self, + page_cache: &PageCache, + pages: &mut BTreeMap<PageOffset, CachePage>, + offset: usize, + len: usize, + copied: usize + ) -> KResult<()> { + Err(EINVAL) + } + + --- + + fn devid(&self) -> KResult<DeviceId> { + Err(EINVAL) + } +} + +#[derive(Debug, Clone)] +pub struct InodeInfo { + pub size: u64, + pub nlink: u64, + + pub uid: u32, + pub gid: u32, + pub perm: Permission, + + pub atime: Instant, + pub ctime: Instant, + pub mtime: Instant, +} + +#[repr(transparent)] +pub struct InodeUse(Arc<Inode>); + +impl InodeUse { + pub fn new( + sb: SbRef<dyn SuperBlock>, + ino: Ino, + format: Format, + info: InodeInfo, + ops: impl InodeOps, + ) -> Self { + let inode = Inode { + sb, + ino, + format, + info: Spin::new(info), + rwsem: RwLock::new(()), + page_cache: Spin::new(Weak::new()), + ops: Box::new(ops), + }; + + Self(Arc::new(inode)) + } + + pub fn sbref(&self) -> SbRef<dyn SuperBlock> { + self.sb.clone() + } + + pub fn sbget(&self) -> KResult<SbUse<dyn SuperBlock>> { + self.sb.get().map(|sb| sb as _) + } + + pub fn get_priv<I>(&self) -> &I + where + I: InodeOps, + { + let ops = (&*self.ops) as &dyn Any; + + ops.downcast_ref() + .expect("InodeUse::private: InodeOps type mismatch") + } + + pub fn get_page_cache(&self) -> Arc<PageCache> { + if let Some(cache) = self.page_cache.lock().upgrade() { + return cache; + } + + // Slow path... + let cache = Arc::new(PageCache::new(self.clone())); + let mut page_cache = self.page_cache.lock(); + if let Some(cache) = page_cache.upgrade() { + return cache; + } + + *page_cache = Arc::downgrade(&cache); + cache + } +} + +impl Clone for InodeUse { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl core::fmt::Debug for InodeUse { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "InodeUse(ino={})", self.ino) + } +} + +impl Deref for InodeUse { + type Target = Inode; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl PartialEq for InodeUse { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} diff --git a/src/kernel/vfs/inode/mod.rs b/src/kernel/vfs/inode/mod.rs new file mode 100644 index 00000000..6f4f041a --- /dev/null +++ b/src/kernel/vfs/inode/mod.rs @@ -0,0 +1,8 @@ +mod ino; +mod inode; +mod ops; +mod statx; + +pub use ino::Ino; +pub use inode::{Inode, InodeInfo, InodeOps, InodeUse}; +pub use ops::{RenameData, WriteOffset}; diff --git a/src/kernel/vfs/inode/ops.rs b/src/kernel/vfs/inode/ops.rs new file mode 100644 index 00000000..7bf00ce5 --- /dev/null +++ b/src/kernel/vfs/inode/ops.rs @@ -0,0 +1,17 @@ +use alloc::sync::Arc; + +use super::inode::InodeUse; +use crate::kernel::vfs::dentry::Dentry; + +pub enum WriteOffset<'end> { + Position(usize), + End(&'end mut usize), +} + +pub struct RenameData<'a, 'b> { + pub old_dentry: &'a Arc<Dentry>, + pub new_dentry: &'b Arc<Dentry>, + pub new_parent: InodeUse, + pub is_exchange: bool, + pub no_replace: bool, +} diff --git a/src/kernel/vfs/inode/statx.rs b/src/kernel/vfs/inode/statx.rs new file mode 100644 index 00000000..feb2a1b5 --- /dev/null +++ b/src/kernel/vfs/inode/statx.rs @@ -0,0 +1,87 @@ +use posix_types::stat::StatX; + +use super::inode::InodeUse; +use crate::kernel::constants::{ + STATX_ATIME, STATX_BLOCKS, STATX_CTIME, STATX_GID, STATX_INO, STATX_MODE, STATX_MTIME, + STATX_NLINK, STATX_SIZE, STATX_TYPE, STATX_UID, +}; +use crate::kernel::vfs::types::Format; +use crate::prelude::KResult; + +impl InodeUse { + pub fn statx(&self, stat: &mut StatX, mask: u32) -> KResult<()> { + let sb = self.sbget()?; + let info = self.info.lock(); + + if mask & STATX_NLINK != 0 { + stat.stx_nlink = info.nlink as _; + stat.stx_mask |= STATX_NLINK; + } + + if mask & STATX_ATIME != 0 { + stat.stx_atime = info.atime.into(); + stat.stx_mask |= STATX_ATIME; + } + + if mask & STATX_MTIME != 0 { + stat.stx_mtime = info.mtime.into(); + stat.stx_mask |= STATX_MTIME; + } + + if mask & STATX_CTIME != 0 { + stat.stx_ctime = info.ctime.into(); + stat.stx_mask |= STATX_CTIME; + } + + if mask & STATX_SIZE != 0 { + stat.stx_size = info.size as _; + stat.stx_mask |= STATX_SIZE; + } + + stat.stx_mode = 0; + if mask & STATX_MODE != 0 { + stat.stx_mode |= info.perm.bits() as u16; + stat.stx_mask |= STATX_MODE; + } + + if mask & STATX_TYPE != 0 { + stat.stx_mode |= self.format.as_raw() as u16; + if let Format::BLK | Format::CHR = self.format { + let devid = self.devid()?; + stat.stx_rdev_major = devid.major as _; + stat.stx_rdev_minor = devid.minor as _; + } + stat.stx_mask |= STATX_TYPE; + } + + if mask & STATX_INO != 0 { + stat.stx_ino = self.ino.as_raw(); + stat.stx_mask |= STATX_INO; + } + + if mask & STATX_BLOCKS != 0 { + stat.stx_blocks = (info.size + 512 - 1) / 512; + stat.stx_blksize = sb.info.io_blksize as _; + stat.stx_mask |= STATX_BLOCKS; + } + + if mask & STATX_UID != 0 { + stat.stx_uid = info.uid; + stat.stx_mask |= STATX_UID; + } + + if mask & STATX_GID != 0 { + stat.stx_gid = info.gid; + stat.stx_mask |= STATX_GID; + } + + let fsdev = sb.info.device_id; + stat.stx_dev_major = fsdev.major as _; + stat.stx_dev_minor = fsdev.minor as _; + + // TODO: support more attributes + stat.stx_attributes_mask = 0; + + Ok(()) + } +} diff --git a/src/kernel/vfs/mod.rs b/src/kernel/vfs/mod.rs index f62cb9b9..5b8eca5a 100644 --- a/src/kernel/vfs/mod.rs +++ b/src/kernel/vfs/mod.rs @@ -1,31 +1,31 @@ -use crate::prelude::*; -use alloc::sync::Arc; -use dentry::Dentry; -use eonix_sync::LazyLock; -use inode::Mode; - pub mod dentry; mod file; pub mod filearray; pub mod inode; pub mod mount; -pub mod vfs; +mod superblock; +pub mod types; -pub use file::{File, FileType, PollEvent, SeekOption, TerminalFile}; +use crate::prelude::*; +use alloc::sync::Arc; +use dentry::Dentry; +use eonix_sync::LazyLock; +use types::Permission; -pub type DevId = u32; +pub use file::{File, FileType, PollEvent, SeekOption, TerminalFile}; +pub use superblock::{SbRef, SbUse, SuperBlock, SuperBlockInfo, SuperBlockLock}; pub struct FsContext { pub fsroot: Arc<Dentry>, pub cwd: Spin<Arc<Dentry>>, - pub umask: Spin<Mode>, + pub umask: Spin<Permission>, } static GLOBAL_FS_CONTEXT: LazyLock<Arc<FsContext>> = LazyLock::new(|| { Arc::new(FsContext { fsroot: Dentry::root().clone(), cwd: Spin::new(Dentry::root().clone()), - umask: Spin::new(Mode::new(0o022)), + umask: Spin::new(Permission::new(0o755)), }) }); diff --git a/src/kernel/vfs/mount.rs b/src/kernel/vfs/mount.rs index 0b38e0c0..fcb5b622 100644 --- a/src/kernel/vfs/mount.rs +++ b/src/kernel/vfs/mount.rs @@ -1,12 +1,19 @@ -use super::{ - dentry::{dcache, Dentry, DROOT}, - inode::Inode, - vfs::Vfs, -}; +use alloc::collections::btree_map::BTreeMap; +use alloc::string::ToString as _; +use alloc::sync::Arc; + +use async_trait::async_trait; +use eonix_macros::define_late_init; +use eonix_sync::LazyLock; + +use super::dentry::{dcache, Dentry, DROOT}; +use super::inode::InodeUse; +use super::{SbUse, SuperBlock}; +use crate::fs::procfs::populate_root; +use crate::io::buf_writeln; use crate::kernel::constants::{EEXIST, ENODEV, ENOTDIR}; +use crate::kernel::task::block_on; use crate::prelude::*; -use alloc::{collections::btree_map::BTreeMap, string::ToString as _, sync::Arc}; -use eonix_sync::LazyLock; pub const MS_RDONLY: u64 = 1 << 0; pub const MS_NOSUID: u64 = 1 << 1; @@ -26,21 +33,24 @@ const MOUNT_FLAGS: [(u64, &str); 6] = [ (MS_LAZYTIME, ",lazytime"), ]; -static MOUNT_CREATORS: Spin<BTreeMap<String, Arc<dyn MountCreator>>> = Spin::new(BTreeMap::new()); +static MOUNT_CREATORS: Spin<BTreeMap<String, Arc<dyn MountCreator>>> = + Spin::new(BTreeMap::new()); static MOUNTS: Spin<Vec<(Arc<Dentry>, MountPointData)>> = Spin::new(vec![]); pub struct Mount { - _vfs: Arc<dyn Vfs>, + sb: SbUse<dyn SuperBlock>, root: Arc<Dentry>, } impl Mount { - pub fn new(mp: &Dentry, vfs: Arc<dyn Vfs>, root_inode: Arc<dyn Inode>) -> KResult<Self> { + pub fn new( + mp: &Dentry, sb: SbUse<dyn SuperBlock>, root_inode: InodeUse, + ) -> KResult<Self> { let root_dentry = Dentry::create(mp.parent().clone(), &mp.get_name()); - root_dentry.save_dir(root_inode)?; + root_dentry.fill(root_inode); Ok(Self { - _vfs: vfs, + sb, root: root_dentry, }) } @@ -53,12 +63,17 @@ impl Mount { unsafe impl Send for Mount {} unsafe impl Sync for Mount {} +#[async_trait] pub trait MountCreator: Send + Sync { fn check_signature(&self, first_block: &[u8]) -> KResult<bool>; - fn create_mount(&self, source: &str, flags: u64, mp: &Arc<Dentry>) -> KResult<Mount>; + async fn create_mount( + &self, source: &str, flags: u64, mp: &Arc<Dentry>, + ) -> KResult<Mount>; } -pub fn register_filesystem(fstype: &str, creator: Arc<dyn MountCreator>) -> KResult<()> { +pub fn register_filesystem( + fstype: &str, creator: Arc<dyn MountCreator>, +) -> KResult<()> { let mut creators = MOUNT_CREATORS.lock(); if !creators.contains_key(fstype) { creators.insert(fstype.to_string(), creator); @@ -77,11 +92,8 @@ struct MountPointData { flags: u64, } -pub fn do_mount( - mountpoint: &Arc<Dentry>, - source: &str, - mountpoint_str: &str, - fstype: &str, +pub async fn do_mount( + mountpoint: &Arc<Dentry>, source: &str, mountpoint_str: &str, fstype: &str, flags: u64, ) -> KResult<()> { let mut flags = flags; @@ -101,7 +113,7 @@ pub fn do_mount( let creators = { MOUNT_CREATORS.lock() }; creators.get(fstype).ok_or(ENODEV)?.clone() }; - let mount = creator.create_mount(source, flags, mountpoint)?; + let mount = creator.create_mount(source, flags, mountpoint).await?; let root_dentry = mount.root().clone(); @@ -139,17 +151,23 @@ fn mount_opts(flags: u64) -> String { out } -pub fn dump_mounts(buffer: &mut dyn core::fmt::Write) { - for (_, mpdata) in MOUNTS.lock().iter() { - dont_check!(writeln!( - buffer, - "{} {} {} {} 0 0", - mpdata.source, - mpdata.mountpoint, - mpdata.fstype, - mount_opts(mpdata.flags) - )) - } +#[define_late_init] +async fn populate_mounts_file() { + populate_root(Arc::from(b"mounts".as_slice()), |buffer| { + for (_, mpdata) in MOUNTS.lock().iter() { + buf_writeln!( + buffer, + "{} {} {} {} 0 0", + mpdata.source, + mpdata.mountpoint, + mpdata.fstype, + mount_opts(mpdata.flags) + )?; + } + + Ok(()) + }) + .await; } impl Dentry { @@ -165,24 +183,21 @@ impl Dentry { .cloned() .expect("tmpfs not registered."); - let mount = creator - .create_mount(&source, mount_flags, &DROOT) - .expect("Failed to create root mount."); + let mount = + block_on(creator.create_mount(&source, mount_flags, &DROOT)) + .expect("Failed to create root mount."); let root_dentry = mount.root().clone(); dcache::d_add(root_dentry.clone()); - MOUNTS.lock().push(( - DROOT.clone(), - MountPointData { - mount, - source, - mountpoint: String::from("/"), - fstype, - flags: mount_flags, - }, - )); + MOUNTS.lock().push((DROOT.clone(), MountPointData { + mount, + source, + mountpoint: String::from("/"), + fstype, + flags: mount_flags, + })); root_dentry }); diff --git a/src/kernel/vfs/superblock.rs b/src/kernel/vfs/superblock.rs new file mode 100644 index 00000000..e3be5cef --- /dev/null +++ b/src/kernel/vfs/superblock.rs @@ -0,0 +1,156 @@ +use alloc::sync::{Arc, Weak}; +use core::any::{Any, TypeId}; +use core::marker::Unsize; +use core::ops::{CoerceUnsized, Deref}; + +use eonix_sync::RwLock; + +use super::types::DeviceId; +use crate::kernel::constants::EIO; +use crate::prelude::KResult; + +pub trait SuperBlock: Any + Send + Sync + 'static {} + +#[derive(Debug, Clone)] +pub struct SuperBlockInfo { + pub io_blksize: u32, + pub device_id: DeviceId, + pub read_only: bool, +} + +pub struct SuperBlockLock(()); + +pub struct SuperBlockComplex<Backend> +where + Backend: SuperBlock + ?Sized, +{ + pub info: SuperBlockInfo, + pub rwsem: RwLock<SuperBlockLock>, + pub backend: Backend, +} + +pub struct SbRef<S>(Weak<SuperBlockComplex<S>>) +where + S: SuperBlock + ?Sized; + +pub struct SbUse<S>(Arc<SuperBlockComplex<S>>) +where + S: SuperBlock + ?Sized; + +impl<S> SbRef<S> +where + S: SuperBlock + ?Sized, +{ + pub fn try_get(&self) -> Option<SbUse<S>> { + self.0.upgrade().map(|arc| SbUse(arc)) + } + + pub fn get(&self) -> KResult<SbUse<S>> { + self.try_get().ok_or(EIO) + } + + pub fn from(sb: &SbUse<S>) -> Self { + SbRef(Arc::downgrade(&sb.0)) + } + + pub fn eq<U>(&self, other: &SbRef<U>) -> bool + where + U: SuperBlock + ?Sized, + { + core::ptr::addr_eq(self.0.as_ptr(), other.0.as_ptr()) + } +} + +impl<S> SbUse<S> +where + S: SuperBlock, +{ + pub fn new(info: SuperBlockInfo, backend: S) -> Self { + Self(Arc::new(SuperBlockComplex { + info, + rwsem: RwLock::new(SuperBlockLock(())), + backend, + })) + } + + pub fn new_cyclic(info: SuperBlockInfo, backend_func: impl FnOnce(SbRef<S>) -> S) -> Self { + Self(Arc::new_cyclic(|weak| SuperBlockComplex { + info, + rwsem: RwLock::new(SuperBlockLock(())), + backend: backend_func(SbRef(weak.clone())), + })) + } +} + +impl<S> SbUse<S> +where + S: SuperBlock + ?Sized, +{ + pub fn get_ref(&self) -> SbRef<S> { + SbRef(Arc::downgrade(&self.0)) + } +} + +impl SbUse<dyn SuperBlock> { + /// Downcast the superblock to a specific type. + /// + /// # Panics + /// Panics if the downcast fails. + pub fn downcast<U: SuperBlock>(self) -> SbUse<U> { + let Self(sb_complex) = self; + if (&sb_complex.backend as &dyn Any).type_id() != TypeId::of::<U>() { + panic!("Downcast failed: type mismatch"); + } + + unsafe { + // SAFETY: We have checked the type above and unsized coercion says + // that Arc<T> has the same layout as Arc<U> if T: Unsize<U>. + SbUse(Arc::from_raw( + Arc::into_raw(sb_complex) as *const SuperBlockComplex<U> + )) + } + } +} + +impl<S> Clone for SbRef<S> +where + S: SuperBlock + ?Sized, +{ + fn clone(&self) -> Self { + SbRef(self.0.clone()) + } +} + +impl<S> Clone for SbUse<S> +where + S: SuperBlock + ?Sized, +{ + fn clone(&self) -> Self { + SbUse(self.0.clone()) + } +} + +impl<T, U> CoerceUnsized<SbRef<U>> for SbRef<T> +where + T: SuperBlock + Unsize<U> + ?Sized, + U: SuperBlock + ?Sized, +{ +} + +impl<T, U> CoerceUnsized<SbUse<U>> for SbUse<T> +where + T: SuperBlock + Unsize<U> + ?Sized, + U: SuperBlock + ?Sized, +{ +} + +impl<S> Deref for SbUse<S> +where + S: SuperBlock + ?Sized, +{ + type Target = SuperBlockComplex<S>; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} diff --git a/src/kernel/vfs/types/device_id.rs b/src/kernel/vfs/types/device_id.rs new file mode 100644 index 00000000..6dd128ee --- /dev/null +++ b/src/kernel/vfs/types/device_id.rs @@ -0,0 +1,25 @@ +use core::fmt::{Debug, Display, Formatter}; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct DeviceId { + pub major: u16, + pub minor: u16, +} + +impl DeviceId { + pub const fn new(major: u16, minor: u16) -> Self { + Self { major, minor } + } +} + +impl Debug for DeviceId { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "DeviceId({:04x}:{:04x})", self.major, self.minor) + } +} + +impl Display for DeviceId { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + write!(f, "{:04x}:{:04x}", self.major, self.minor) + } +} diff --git a/src/kernel/vfs/types/mod.rs b/src/kernel/vfs/types/mod.rs new file mode 100644 index 00000000..4a7505f7 --- /dev/null +++ b/src/kernel/vfs/types/mod.rs @@ -0,0 +1,5 @@ +mod device_id; +mod mode; + +pub use device_id::DeviceId; +pub use mode::{Format, Mode, Permission}; diff --git a/src/kernel/vfs/types/mode.rs b/src/kernel/vfs/types/mode.rs new file mode 100644 index 00000000..a58c8215 --- /dev/null +++ b/src/kernel/vfs/types/mode.rs @@ -0,0 +1,172 @@ +use crate::kernel::{ + constants::{S_IFBLK, S_IFCHR, S_IFDIR, S_IFLNK, S_IFMT, S_IFREG}, + syscall::{FromSyscallArg, SyscallRetVal}, +}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Mode(u32); + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Format { + REG, + DIR, + LNK, + BLK, + CHR, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct Permission(u32); + +impl Mode { + pub const fn new(bits: u32) -> Self { + Self(bits) + } + + pub const fn is_blk(&self) -> bool { + (self.0 & S_IFMT) == S_IFBLK + } + + pub const fn is_chr(&self) -> bool { + (self.0 & S_IFMT) == S_IFCHR + } + + pub const fn bits(&self) -> u32 { + self.0 + } + + pub const fn format_bits(&self) -> u32 { + self.0 & S_IFMT + } + + pub const fn non_format_bits(&self) -> u32 { + self.0 & !S_IFMT + } + + pub fn format(&self) -> Format { + match self.try_format() { + None => panic!("unknown format bits: {:#o}", self.format_bits()), + Some(format) => format, + } + } + + pub fn try_format(&self) -> Option<Format> { + match self.format_bits() { + S_IFREG => Some(Format::REG), + S_IFDIR => Some(Format::DIR), + S_IFLNK => Some(Format::LNK), + S_IFBLK => Some(Format::BLK), + S_IFCHR => Some(Format::CHR), + _ => None, + } + } + + pub fn perm(&self) -> Permission { + Permission::new(self.non_format_bits()) + } + + pub const fn set_perm(&mut self, perm: Permission) { + self.0 = self.format_bits() | perm.bits(); + } +} + +impl Format { + pub const fn as_raw(&self) -> u32 { + match self { + Self::REG => S_IFREG, + Self::DIR => S_IFDIR, + Self::LNK => S_IFLNK, + Self::BLK => S_IFBLK, + Self::CHR => S_IFCHR, + } + } +} + +impl Permission { + const RWX: [&str; 8] = ["---", "--x", "-w-", "-wx", "r--", "r-x", "rw-", "rwx"]; + + pub const fn new(perm_bits: u32) -> Self { + Self(perm_bits & 0o7777) + } + + pub const fn bits(&self) -> u32 { + self.0 + } + + pub const fn mask_with(&self, mask: Self) -> Self { + Self(self.0 & mask.0) + } +} + +impl core::fmt::Debug for Mode { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.non_format_bits() & !0o777 { + 0 => write!( + f, + "Mode({format:?}, {perm:?})", + format = self.try_format(), + perm = Permission::new(self.non_format_bits()), + )?, + rem => write!( + f, + "Mode({format:?}, {perm:?}, rem={rem:#x})", + format = self.try_format(), + perm = Permission::new(self.non_format_bits()) + )?, + } + + Ok(()) + } +} + +impl core::fmt::Debug for Format { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::REG => write!(f, "REG"), + Self::DIR => write!(f, "DIR"), + Self::LNK => write!(f, "LNK"), + Self::BLK => write!(f, "BLK"), + Self::CHR => write!(f, "CHR"), + } + } +} + +impl core::fmt::Debug for Permission { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let owner = self.0 >> 6 & 0o7; + let group = self.0 >> 3 & 0o7; + let other = self.0 & 0o7; + + write!( + f, + "{}{}{}", + Self::RWX[owner as usize], + Self::RWX[group as usize], + Self::RWX[other as usize] + ) + } +} + +impl FromSyscallArg for Mode { + fn from_arg(value: usize) -> Self { + Mode::new(value as u32) + } +} + +impl SyscallRetVal for Mode { + fn into_retval(self) -> Option<usize> { + Some(self.bits() as usize) + } +} + +impl FromSyscallArg for Permission { + fn from_arg(value: usize) -> Self { + Permission::new(value as u32) + } +} + +impl SyscallRetVal for Permission { + fn into_retval(self) -> Option<usize> { + Some(self.bits() as usize) + } +} diff --git a/src/kernel/vfs/vfs.rs b/src/kernel/vfs/vfs.rs deleted file mode 100644 index ee66f0b6..00000000 --- a/src/kernel/vfs/vfs.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::prelude::*; - -use super::DevId; - -#[allow(dead_code)] -pub trait Vfs: Send + Sync + AsAny { - fn io_blksize(&self) -> usize; - fn fs_devid(&self) -> DevId; - fn is_read_only(&self) -> bool; -} diff --git a/src/kernel_init.rs b/src/kernel_init.rs index 3d8be90f..2259f6cf 100644 --- a/src/kernel_init.rs +++ b/src/kernel_init.rs @@ -1,35 +1,27 @@ -use crate::kernel::mem::{GlobalPageAlloc, RawPage}; -use eonix_hal::{ - bootstrap::BootStrapData, - mm::{ArchMemory, ArchPagingMode, GLOBAL_PAGE_TABLE}, - traits::mm::Memory, -}; -use eonix_mm::{ - address::{Addr as _, AddrOps as _, VAddr, VRange}, - page_table::{PageAttribute, PagingMode as _, PTE}, - paging::{Page as GenericPage, PAGE_SIZE, PFN}, -}; - -pub fn setup_memory(data: &mut BootStrapData) { - let addr_max = ArchMemory::present_ram() - .map(|range| range.end()) - .max() - .expect("No free memory"); +use eonix_hal::arch_exported::mm::{ArchPagingMode, PageAccessImpl}; +use eonix_hal::bootstrap::BootStrapData; +use eonix_hal::mm::{ArchMemory, BasicPageAllocRef, GLOBAL_PAGE_TABLE}; +use eonix_hal::traits::mm::Memory; +use eonix_mm::address::{Addr as _, AddrOps as _, VAddr, VRange}; +use eonix_mm::page_table::{PageAttribute, PageTable, PTE}; +use eonix_mm::paging::{Folio as _, FrameAlloc, PAGE_SIZE, PFN}; - let pfn_max = PFN::from(addr_max.ceil()); - let len_bytes_page_array = usize::from(pfn_max) * size_of::<RawPage>(); - let count_pages = len_bytes_page_array.div_ceil(PAGE_SIZE); +use crate::kernel::mem::{GlobalPageAlloc, RawPage}; - let alloc = data.get_alloc().unwrap(); +fn setup_kernel_page_array(alloc: BasicPageAllocRef, count_pages: usize) { + // TODO: This should be done by the global Zone + let global_page_table = PageTable::<ArchPagingMode, _, _>::new( + GLOBAL_PAGE_TABLE.clone(), + alloc.clone(), + PageAccessImpl, + ); // Map kernel page array. const V_KERNEL_PAGE_ARRAY_START: VAddr = VAddr::from(0xffffff8040000000); - for pte in GLOBAL_PAGE_TABLE.iter_kernel_in( - VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages), - ArchPagingMode::LEVELS, - &alloc, - ) { + let range = + VRange::from(V_KERNEL_PAGE_ARRAY_START).grow(PAGE_SIZE * count_pages); + for pte in global_page_table.iter_kernel(range) { let attr = PageAttribute::PRESENT | PageAttribute::WRITE | PageAttribute::READ @@ -37,22 +29,39 @@ pub fn setup_memory(data: &mut BootStrapData) { | PageAttribute::ACCESSED | PageAttribute::DIRTY; - let page = GenericPage::alloc_in(&alloc); + let page = alloc.alloc().unwrap(); pte.set(page.into_raw(), attr.into()); } + // TODO!!!: Construct the global zone with all present ram. + // for range in ArchMemory::present_ram() { + // GlobalPageAlloc::mark_present(range); + // } + unsafe { // SAFETY: We've just mapped the area with sufficient length. core::ptr::write_bytes( - V_KERNEL_PAGE_ARRAY_START.addr() as *mut (), + V_KERNEL_PAGE_ARRAY_START.addr() as *mut u8, 0, count_pages * PAGE_SIZE, ); } - for range in ArchMemory::present_ram() { - GlobalPageAlloc::mark_present(range); - } + core::mem::forget(global_page_table); +} + +pub fn setup_memory(data: &mut BootStrapData) { + let addr_max = ArchMemory::present_ram() + .map(|range| range.end()) + .max() + .expect("No free memory"); + + let pfn_max = PFN::from(addr_max.ceil()); + let len_bytes_page_array = usize::from(pfn_max) * size_of::<RawPage>(); + let count_pages = len_bytes_page_array.div_ceil(PAGE_SIZE); + + let alloc = data.get_alloc().unwrap(); + setup_kernel_page_array(alloc, count_pages); if let Some(early_alloc) = data.take_alloc() { for range in early_alloc.into_iter() { diff --git a/src/lib.rs b/src/lib.rs index 80d24c28..a9087beb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,15 +2,16 @@ #![no_main] #![feature(allocator_api)] #![feature(c_size_t)] -#![feature(concat_idents)] +#![feature(coerce_unsized)] #![feature(arbitrary_self_types)] #![feature(get_mut_unchecked)] #![feature(macro_metavar_expr)] +#![feature(unsize)] extern crate alloc; -#[cfg(any(target_arch = "riscv64", target_arch = "x86_64"))] -extern crate unwinding; +#[macro_use] +extern crate static_assertions; mod driver; mod fs; @@ -19,97 +20,39 @@ mod io; mod kernel; mod kernel_init; mod net; -#[cfg(any(target_arch = "riscv64", target_arch = "x86_64"))] mod panic; mod path; mod prelude; mod rcu; mod sync; -use crate::kernel::task::alloc_pid; -use alloc::{ffi::CString, sync::Arc}; -use core::{ - hint::spin_loop, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, -}; -use eonix_hal::{ - arch_exported::bootstrap::shutdown, - context::TaskContext, - processor::{halt, CPU, CPU_COUNT}, - traits::{context::RawTaskContext, trap::IrqState}, - trap::disable_irqs_save, -}; +use alloc::ffi::CString; +use core::hint::spin_loop; +use core::sync::atomic::{AtomicBool, Ordering}; + +use eonix_hal::processor::CPU; +use eonix_hal::symbol_addr; +use eonix_hal::traits::trap::{IrqState, RawTrapContext, TrapReturn}; +use eonix_hal::trap::{disable_irqs_save, TrapContext}; +use eonix_log::println_debug; use eonix_mm::address::PRange; -use eonix_runtime::{executor::Stack, scheduler::RUNTIME}; -use kernel::{ - mem::GlobalPageAlloc, - task::{KernelStack, ProcessBuilder, ProcessList, ProgramLoader, ThreadBuilder}, - vfs::{ - dentry::Dentry, - inode::Mode, - mount::{do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY}, - FsContext, - }, - CharDevice, +use eonix_runtime::scheduler::RUNTIME; +use kernel::mem::GlobalPageAlloc; +use kernel::task::{KernelStack, ProcessList, ProgramLoader}; +use kernel::vfs::dentry::Dentry; +use kernel::vfs::mount::{ + do_mount, MS_NOATIME, MS_NODEV, MS_NOSUID, MS_RDONLY, }; +use kernel::vfs::types::Permission; +use kernel::vfs::FsContext; +use kernel::{shutdown_system, CharDevice}; use kernel_init::setup_memory; use path::Path; use prelude::*; -#[cfg(any(target_arch = "riscv64", target_arch = "loongarch64"))] -fn do_panic() -> ! { - #[cfg(target_arch = "riscv64")] - panic::stack_trace(); - - shutdown(); -} - -#[cfg(not(any(target_arch = "riscv64", target_arch = "loongarch64")))] -fn do_panic() -> ! { - // Spin forever. - loop { - spin_loop(); - } -} - -#[panic_handler] -fn panic(info: &core::panic::PanicInfo) -> ! { - if let Some(location) = info.location() { - println_fatal!( - "panicked at {}:{}:{}", - location.file(), - location.line(), - location.column() - ); - } else { - println_fatal!("panicked at <UNKNOWN>"); - } - println_fatal!(); - println_fatal!("{}", info.message()); - - do_panic() -} +use crate::kernel::{run_late_init, run_late_init_async}; static BSP_OK: AtomicBool = AtomicBool::new(false); -static CPU_SHUTTING_DOWN: AtomicUsize = AtomicUsize::new(0); - -fn shutdown_system() -> ! { - let cpu_count = CPU_COUNT.load(Ordering::Relaxed); - - if CPU_SHUTTING_DOWN.fetch_add(1, Ordering::AcqRel) + 1 == cpu_count { - println_info!("All CPUs are shutting down. Gracefully powering off..."); - shutdown(); - } else { - println_info!( - "CPU {} is shutting down. Waiting for other CPUs...", - CPU::local().cpuid() - ); - - loop { - halt(); - } - } -} #[eonix_hal::main] fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! { @@ -126,20 +69,15 @@ fn kernel_init(mut data: eonix_hal::bootstrap::BootStrapData) -> ! { drop(data); - let mut ctx = TaskContext::new(); - let stack_bottom = { - let stack = KernelStack::new(); - let bottom = stack.get_bottom().addr().get(); - core::mem::forget(stack); + let mut ctx = TrapContext::new(true, false); + let stack = KernelStack::new(); - bottom - }; - ctx.set_interrupt_enabled(true); - ctx.set_program_counter(standard_main as usize); - ctx.set_stack_pointer(stack_bottom); + ctx.set_kernel_call_frame(symbol_addr!(standard_main), &stack, None, &[]); + + core::mem::forget(stack); unsafe { - TaskContext::switch_to_noreturn(&mut ctx); + ctx.trap_return_noreturn(); } } @@ -152,20 +90,15 @@ fn kernel_ap_main(_stack_range: PRange) -> ! { println_debug!("AP{} started", CPU::local().cpuid()); - let mut ctx = TaskContext::new(); - let stack_bottom = { - let stack = KernelStack::new(); - let bottom = stack.get_bottom().addr().get(); - core::mem::forget(stack); + let mut ctx = TrapContext::new(true, false); + let stack = KernelStack::new(); - bottom - }; - ctx.set_interrupt_enabled(true); - ctx.set_program_counter(standard_main as usize); - ctx.set_stack_pointer(stack_bottom); + ctx.set_kernel_call_frame(symbol_addr!(standard_main), &stack, None, &[]); + + core::mem::forget(stack); unsafe { - TaskContext::switch_to_noreturn(&mut ctx); + ctx.trap_return_noreturn(); } } @@ -192,16 +125,16 @@ async fn init_process(early_kstack: PRange) { { // We might want the serial initialized as soon as possible. driver::serial::init().unwrap(); - driver::e1000e::register_e1000e_driver(); - driver::ahci::register_ahci_driver(); + driver::e1000e::register_e1000e_driver().await; + driver::ahci::register_ahci_driver().await; } #[cfg(target_arch = "riscv64")] { driver::serial::init().unwrap(); driver::virtio::init_virtio_devices(); - driver::e1000e::register_e1000e_driver(); - driver::ahci::register_ahci_driver(); + driver::e1000e::register_e1000e_driver().await; + driver::ahci::register_ahci_driver().await; driver::goldfish_rtc::probe(); } @@ -209,21 +142,30 @@ async fn init_process(early_kstack: PRange) { { driver::serial::init().unwrap(); driver::virtio::init_virtio_devices(); - driver::e1000e::register_e1000e_driver(); - driver::ahci::register_ahci_driver(); + driver::e1000e::register_e1000e_driver().await; + driver::ahci::register_ahci_driver().await; } fs::tmpfs::init(); - fs::procfs::init(); + fs::procfs::init().await; fs::fat32::init(); - fs::ext4::init(); + // fs::ext4::init(); + + run_late_init(); + run_late_init_async().await; let load_info = { // mount fat32 /mnt directory let fs_context = FsContext::global(); - let mnt_dir = Dentry::open(fs_context, Path::new(b"/mnt/").unwrap(), true).unwrap(); + let mnt_dir = + Dentry::open(fs_context, Path::new(b"/mnt/").unwrap(), true) + .await + .unwrap(); - mnt_dir.mkdir(Mode::new(0o755)).unwrap(); + mnt_dir + .mkdir(Permission::new(0o755)) + .await + .expect("Failed to create /mnt directory"); do_mount( &mnt_dir, @@ -232,14 +174,18 @@ async fn init_process(early_kstack: PRange) { "fat32", MS_RDONLY | MS_NOATIME | MS_NODEV | MS_NOSUID, ) + .await .unwrap(); - let init_names = [&b"/init"[..], &b"/sbin/init"[..], &b"/mnt/initsh"[..]]; + let init_names = + [&b"/init"[..], &b"/sbin/init"[..], &b"/mnt/initsh"[..]]; let mut init_name = None; let mut init = None; for name in init_names { - if let Ok(dentry) = Dentry::open(fs_context, Path::new(name).unwrap(), true) { + if let Ok(dentry) = + Dentry::open(fs_context, Path::new(name).unwrap(), true).await + { if dentry.is_valid() { init_name = Some(CString::new(name).unwrap()); init = Some(dentry); @@ -261,27 +207,12 @@ async fn init_process(early_kstack: PRange) { ]; ProgramLoader::parse(fs_context, init_name, init.clone(), argv, envp) + .await .expect("Failed to parse init program") .load() .await .expect("Failed to load init program") }; - let thread_builder = ThreadBuilder::new() - .name(Arc::from(&b"busybox"[..])) - .entry(load_info.entry_ip, load_info.sp); - - let mut process_list = ProcessList::get().write().await; - let (thread, process) = ProcessBuilder::new() - .pid(alloc_pid()) - .mm_list(load_info.mm_list) - .thread_builder(thread_builder) - .build(&mut process_list); - - process_list.set_init_process(process); - - // TODO!!!: Remove this. - thread.files.open_console(); - - RUNTIME.spawn(thread.run()); + ProcessList::sys_init(load_info).await; } diff --git a/src/panic.rs b/src/panic.rs index 3c9c5f34..75471169 100644 --- a/src/panic.rs +++ b/src/panic.rs @@ -1,16 +1,49 @@ -use core::ffi::c_void; +use core::panic::PanicInfo; use eonix_log::println_fatal; -use unwinding::abi::{ - UnwindContext, UnwindReasonCode, _Unwind_Backtrace, _Unwind_GetIP, _Unwind_GetRegionStart, -}; -pub fn stack_trace() { +use crate::kernel::shutdown_system; + +#[panic_handler] +fn panic(info: &PanicInfo) -> ! { + if let Some(location) = info.location() { + println_fatal!( + "panicked at {}:{}:{}", + location.file(), + location.line(), + location.column() + ); + } else { + println_fatal!("panicked at <UNKNOWN>"); + } + println_fatal!(); + println_fatal!("{}", info.message()); + + #[cfg(arch_has_stacktrace)] + stacktrace::print_stacktrace(); + + shutdown_system(); +} + +#[cfg(arch_has_stacktrace)] +mod stacktrace { + extern crate unwinding; + + use core::ffi::c_void; + + use eonix_log::println_fatal; + use unwinding::abi::{ + UnwindContext, UnwindReasonCode, _Unwind_Backtrace, _Unwind_GetIP, + _Unwind_GetRegionStart, + }; + struct CallbackData { counter: usize, } - extern "C" fn callback(unwind_ctx: &UnwindContext<'_>, arg: *mut c_void) -> UnwindReasonCode { + extern "C" fn unwind_print( + unwind_ctx: &UnwindContext<'_>, arg: *mut c_void, + ) -> UnwindReasonCode { let data = unsafe { &mut *(arg as *mut CallbackData) }; data.counter += 1; @@ -24,6 +57,14 @@ pub fn stack_trace() { UnwindReasonCode::NO_REASON } - let mut data = CallbackData { counter: 0 }; - _Unwind_Backtrace(callback, &raw mut data as *mut c_void); + pub fn print_stacktrace() { + println_fatal!("--------------8< CUT HERE 8<--------------"); + println_fatal!("Stacktrace:"); + println_fatal!(); + + let mut data = CallbackData { counter: 0 }; + _Unwind_Backtrace(unwind_print, &raw mut data as *mut c_void); + + println_fatal!("--------------8< CUT HERE 8<--------------"); + } } diff --git a/src/path.rs b/src/path.rs index 8b740095..47b9a4b6 100644 --- a/src/path.rs +++ b/src/path.rs @@ -1,34 +1,32 @@ -use crate::{kernel::constants::ENOENT, prelude::*}; use core::fmt::{self, Debug, Formatter}; -pub struct Path<'lt> { - all: &'lt [u8], +use crate::kernel::constants::ENOENT; +use crate::prelude::*; + +#[repr(transparent)] +pub struct Path { + all: [u8], } pub struct PathIterator<'lt> { rem: &'lt [u8], } -#[allow(dead_code)] -impl<'lt> Path<'lt> { - pub fn new(all: &'lt [u8]) -> KResult<Self> { +impl Path { + pub fn new(all: &[u8]) -> KResult<&Self> { if all.is_empty() { Err(ENOENT) } else { - Ok(Self { all }) + Ok(unsafe { &*(all as *const [u8] as *const Path) }) } } - pub fn from_str(all: &'lt str) -> KResult<Self> { - Self::new(all.as_bytes()) - } - pub fn is_absolute(&self) -> bool { self.all.starts_with(&['/' as u8]) } - pub fn iter(&self) -> PathIterator<'lt> { - PathIterator::new(self.all) + pub fn iter(&self) -> PathIterator<'_> { + PathIterator::new(&self.all) } } @@ -46,11 +44,17 @@ pub enum PathComponent<'lt> { Parent, } +impl PathIterator<'_> { + pub fn is_empty(&self) -> bool { + self.rem.is_empty() + } +} + impl<'lt> Iterator for PathIterator<'lt> { type Item = PathComponent<'lt>; fn next(&mut self) -> Option<Self::Item> { - if self.rem.is_empty() { + if self.is_empty() { return None; } @@ -71,16 +75,16 @@ impl<'lt> Iterator for PathIterator<'lt> { self.rem = rem; match cur { - cur if cur.is_empty() => Some(PathComponent::TrailingEmpty), - cur if cur == b"." => Some(PathComponent::Current), - cur if cur == b".." => Some(PathComponent::Parent), - cur => Some(PathComponent::Name(cur)), + b"" => Some(PathComponent::TrailingEmpty), + b"." => Some(PathComponent::Current), + b".." => Some(PathComponent::Parent), + name => Some(PathComponent::Name(name)), } } } -impl Debug for Path<'_> { +impl Debug for Path { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "Path({:?})", self.all) + write!(f, "Path({:?})", &self.all) } } diff --git a/src/prelude.rs b/src/prelude.rs index b3dbe2ce..097dac6b 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,51 +1,12 @@ #[allow(dead_code)] pub type KResult<T> = Result<T, u32>; -macro_rules! dont_check { - ($arg:expr) => { - match $arg { - Ok(_) => (), - Err(_) => (), - } - }; -} - -pub(crate) use dont_check; - -pub(crate) use crate::kernel::console::{ - print, println, println_debug, println_fatal, println_info, println_trace, println_warn, -}; - -pub(crate) use alloc::{boxed::Box, string::String, vec, vec::Vec}; - -pub(crate) use core::{any::Any, fmt::Write, marker::PhantomData, str}; +pub(crate) use alloc::boxed::Box; +pub(crate) use alloc::string::String; +pub(crate) use alloc::vec; +pub(crate) use alloc::vec::Vec; +pub(crate) use core::fmt::Write; +pub(crate) use core::marker::PhantomData; +pub(crate) use core::str; pub use crate::sync::Spin; - -#[allow(dead_code)] -pub trait AsAny: Send + Sync { - fn as_any(&self) -> &dyn Any; - fn as_any_mut(&mut self) -> &mut dyn Any; -} - -macro_rules! impl_any { - ($t:ty) => { - impl AsAny for $t { - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - } - }; -} - -macro_rules! addr_of_mut_field { - ($pointer:expr, $field:ident) => { - core::ptr::addr_of_mut!((*$pointer).$field) - }; -} - -pub(crate) use {addr_of_mut_field, impl_any}; diff --git a/src/rcu.rs b/src/rcu.rs index c1645d33..7fa5ee70 100644 --- a/src/rcu.rs +++ b/src/rcu.rs @@ -1,38 +1,45 @@ -use crate::{kernel::task::block_on, prelude::*}; use alloc::sync::Arc; -use core::{ - ops::Deref, - ptr::NonNull, - sync::atomic::{AtomicPtr, Ordering}, -}; +use core::ops::Deref; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicPtr, Ordering}; + +use arcref::ArcRef; +use eonix_preempt::PreemptGuard; use eonix_runtime::scheduler::RUNTIME; -use eonix_sync::{Mutex, RwLock, RwLockReadGuard}; +use eonix_sync::{RwLock, RwLockReadGuard}; use pointers::BorrowedArc; +use crate::kernel::task::block_on; +use crate::prelude::*; + +/// The RCU Read Lock. Holding a reference to an instance of the struct assures +/// you that any RCU protected data would not be dropped. +/// +/// The struct cannot be created directly. Instead, use [`rcu_read_lock()`]. +#[derive(Debug)] +pub struct RCUReadLock(); + +pub struct RCUReadGuardNew { + guard: RwLockReadGuard<'static, RCUReadLock>, + _disable_preempt: PreemptGuard<()>, +} + pub struct RCUReadGuard<'data, T: 'data> { value: T, - _guard: RwLockReadGuard<'data, ()>, + _guard: RCUReadGuardNew, _phantom: PhantomData<&'data T>, } -static GLOBAL_RCU_SEM: RwLock<()> = RwLock::new(()); +static GLOBAL_RCU_SEM: RwLock<RCUReadLock> = RwLock::new(RCUReadLock()); impl<'data, T> RCUReadGuard<'data, BorrowedArc<'data, T>> { fn lock(value: BorrowedArc<'data, T>) -> Self { Self { value, - _guard: block_on(GLOBAL_RCU_SEM.read()), + _guard: rcu_read_lock(), _phantom: PhantomData, } } - - pub fn borrow(&self) -> BorrowedArc<'data, T> { - unsafe { - BorrowedArc::from_raw(NonNull::new_unchecked( - &raw const *self.value.borrow() as *mut T - )) - } - } } impl<'data, T: 'data> Deref for RCUReadGuard<'data, T> { @@ -63,17 +70,14 @@ pub trait RCUNode<MySelf> { pub struct RCUList<T: RCUNode<T>> { head: AtomicPtr<T>, - - reader_lock: RwLock<()>, - update_lock: Mutex<()>, + update_lock: Spin<()>, } impl<T: RCUNode<T>> RCUList<T> { pub const fn new() -> Self { Self { head: AtomicPtr::new(core::ptr::null_mut()), - reader_lock: RwLock::new(()), - update_lock: Mutex::new(()), + update_lock: Spin::new(()), } } @@ -87,9 +91,10 @@ impl<T: RCUNode<T>> RCUList<T> { new_node.rcu_next().store(old_head, Ordering::Release); if let Some(old_head) = unsafe { old_head.as_ref() } { - old_head - .rcu_prev() - .store(Arc::into_raw(new_node.clone()) as *mut _, Ordering::Release); + old_head.rcu_prev().store( + Arc::into_raw(new_node.clone()) as *mut _, + Ordering::Release, + ); } self.head @@ -109,15 +114,14 @@ impl<T: RCUNode<T>> RCUList<T> { } { - let prev_next = - unsafe { prev.as_ref().map(|rcu| rcu.rcu_next()) }.unwrap_or(&self.head); + let prev_next = unsafe { prev.as_ref().map(|rcu| rcu.rcu_next()) } + .unwrap_or(&self.head); let me = prev_next.swap(next, Ordering::AcqRel); debug_assert!(me == Arc::as_ptr(&node) as *mut _); unsafe { Arc::from_raw(me) }; } - let _lck = self.reader_lock.write(); node.rcu_prev() .store(core::ptr::null_mut(), Ordering::Release); node.rcu_next() @@ -134,25 +138,28 @@ impl<T: RCUNode<T>> RCUList<T> { new_node.rcu_next().store(next, Ordering::Release); { - let prev_next = - unsafe { prev.as_ref().map(|rcu| rcu.rcu_next()) }.unwrap_or(&self.head); + let prev_next = unsafe { prev.as_ref().map(|rcu| rcu.rcu_next()) } + .unwrap_or(&self.head); - let old = prev_next.swap(Arc::into_raw(new_node.clone()) as *mut _, Ordering::AcqRel); + let old = prev_next.swap( + Arc::into_raw(new_node.clone()) as *mut _, + Ordering::AcqRel, + ); debug_assert!(old == Arc::as_ptr(&old_node) as *mut _); unsafe { Arc::from_raw(old) }; } if let Some(next) = unsafe { next.as_ref() } { - let old = next - .rcu_prev() - .swap(Arc::into_raw(new_node.clone()) as *mut _, Ordering::AcqRel); + let old = next.rcu_prev().swap( + Arc::into_raw(new_node.clone()) as *mut _, + Ordering::AcqRel, + ); debug_assert!(old == Arc::as_ptr(&old_node) as *mut _); unsafe { Arc::from_raw(old) }; } - let _lck = self.reader_lock.write(); old_node .rcu_prev() .store(core::ptr::null_mut(), Ordering::Release); @@ -161,36 +168,39 @@ impl<T: RCUNode<T>> RCUList<T> { .store(core::ptr::null_mut(), Ordering::Release); } - pub fn iter(&self) -> RCUIterator<T> { - let _lck = block_on(self.reader_lock.read()); - + pub fn iter<'a, 'r>( + &'a self, _lock: &'r RCUReadLock, + ) -> RCUIterator<'a, 'r, T> { RCUIterator { - // SAFETY: We have a read lock, so the node is still alive. - cur: NonNull::new(self.head.load(Ordering::SeqCst)), - _lock: _lck, + cur: NonNull::new(self.head.load(Ordering::Acquire)), + _phantom: PhantomData, } } } -pub struct RCUIterator<'lt, T: RCUNode<T>> { +pub struct RCUIterator<'list, 'rcu, T: RCUNode<T>> { cur: Option<NonNull<T>>, - _lock: RwLockReadGuard<'lt, ()>, + _phantom: PhantomData<(&'list (), &'rcu ())>, } -impl<'lt, T: RCUNode<T>> Iterator for RCUIterator<'lt, T> { - type Item = BorrowedArc<'lt, T>; +impl<'rcu, T: RCUNode<T>> Iterator for RCUIterator<'_, 'rcu, T> { + type Item = ArcRef<'rcu, T>; fn next(&mut self) -> Option<Self::Item> { - match self.cur { - None => None, - Some(pointer) => { - // SAFETY: We have a read lock, so the node is still alive. - let reference = unsafe { pointer.as_ref() }; - - self.cur = NonNull::new(reference.rcu_next().load(Ordering::SeqCst)); - Some(unsafe { BorrowedArc::from_raw(pointer) }) + self.cur.map(|pointer| { + let reference = unsafe { + // SAFETY: We have the read lock so the node is still alive. + pointer.as_ref() + }; + + self.cur = + NonNull::new(reference.rcu_next().load(Ordering::Acquire)); + + unsafe { + // SAFETY: We have the read lock so the node is still alive. + ArcRef::new_unchecked(pointer.as_ptr()) } - } + }) } } @@ -228,21 +238,25 @@ where } pub fn load<'lt>(&self) -> Option<RCUReadGuard<'lt, BorrowedArc<'lt, T>>> { + // BUG: We should acquire the lock before loading the pointer NonNull::new(self.0.load(Ordering::Acquire)) .map(|p| RCUReadGuard::lock(unsafe { BorrowedArc::from_raw(p) })) } - pub fn load_protected<'a, U: 'a>( - &self, - _guard: &RCUReadGuard<'a, U>, - ) -> Option<BorrowedArc<'a, T>> { - NonNull::new(self.0.load(Ordering::Acquire)).map(|p| unsafe { BorrowedArc::from_raw(p) }) + pub fn dereference<'r, 'a: 'r>( + &self, _lock: &'a RCUReadLock, + ) -> Option<ArcRef<'r, T>> { + NonNull::new(self.0.load(Ordering::Acquire)).map(|p| unsafe { + // SAFETY: We have a read lock, so the node is still alive. + ArcRef::new_unchecked(p.as_ptr()) + }) } /// # Safety /// Caller must ensure no writers are updating the pointer. pub unsafe fn load_locked<'lt>(&self) -> Option<BorrowedArc<'lt, T>> { - NonNull::new(self.0.load(Ordering::Acquire)).map(|p| unsafe { BorrowedArc::from_raw(p) }) + NonNull::new(self.0.load(Ordering::Acquire)) + .map(|p| unsafe { BorrowedArc::from_raw(p) }) } /// # Safety @@ -289,3 +303,18 @@ where } } } + +impl Deref for RCUReadGuardNew { + type Target = RCUReadLock; + + fn deref(&self) -> &Self::Target { + &self.guard + } +} + +pub fn rcu_read_lock() -> RCUReadGuardNew { + RCUReadGuardNew { + guard: block_on(GLOBAL_RCU_SEM.read()), + _disable_preempt: PreemptGuard::new(()), + } +} diff --git a/src/sync/arcswap.rs b/src/sync/arcswap.rs index fb8219b2..7421659f 100644 --- a/src/sync/arcswap.rs +++ b/src/sync/arcswap.rs @@ -1,9 +1,8 @@ use alloc::sync::Arc; -use core::{ - fmt::{self, Debug, Formatter}, - ptr::NonNull, - sync::atomic::{AtomicPtr, Ordering}, -}; +use core::fmt::{self, Debug, Formatter}; +use core::ptr::NonNull; +use core::sync::atomic::{AtomicPtr, Ordering}; + use pointers::BorrowedArc; unsafe impl<T> Send for ArcSwap<T> where T: Send + Sync {} @@ -33,7 +32,7 @@ impl<T> ArcSwap<T> { } } - pub fn borrow(&self) -> BorrowedArc<T> { + pub fn borrow(&self) -> BorrowedArc<'_, T> { unsafe { BorrowedArc::from_raw( NonNull::new(self.pointer.load(Ordering::Acquire)) diff --git a/user-programs/init_script_loongarch64.sh b/user-programs/init_script_loongarch64-virt.sh similarity index 100% rename from user-programs/init_script_loongarch64.sh rename to user-programs/init_script_loongarch64-virt.sh diff --git a/user-programs/init_script_riscv64-virt.sh b/user-programs/init_script_riscv64-virt.sh new file mode 100644 index 00000000..f67e2a27 --- /dev/null +++ b/user-programs/init_script_riscv64-virt.sh @@ -0,0 +1,115 @@ +#!/mnt/busybox sh + +BUSYBOX=/mnt/busybox +TERMINAL=/dev/ttyS0 +VERBOSE= + +error() { + printf "\033[91merror: \033[0m%s\n" "$1" >&2 +} + +warn() { + printf "\033[93mwarn : \033[0m%s\n" "$1" >&2 +} + +info() { + printf "\033[92minfo : \033[0m%s\n" "$1" >&2 +} + +die() { + error "$1" && freeze +} + +freeze() { + info "freezing..." >&2 + while true; do + : + done + + exit 1 +} + +unrecoverable() { + die "unrecoverable error occurred. check the message above." +} + +busybox() { + $BUSYBOX "$@" +} + +trap unrecoverable EXIT + +set -euo pipefail + +if [ -n "$VERBOSE" ]; then + set -x +fi + +busybox mkdir -p /dev + +busybox mknod -m 666 /dev/console c 5 1 +busybox mknod -m 666 /dev/null c 1 3 +busybox mknod -m 666 /dev/zero c 1 5 +busybox mknod -m 666 /dev/vda b 8 0 +busybox mknod -m 666 /dev/vda1 b 8 1 +busybox mknod -m 666 /dev/vdb b 8 16 +busybox mknod -m 666 /dev/ttyS0 c 4 64 +busybox mknod -m 666 /dev/ttyS1 c 4 65 + +exec < "$TERMINAL" +exec > "$TERMINAL" 2>&1 + +info "deploying busybox..." + +busybox mkdir -p /bin /lib +busybox --install -s /bin + +info "done" + +export PATH="/bin" + +mkdir -p /etc /root /proc +mount -t procfs proc proc + +cat > /etc/passwd <<EOF +root:x:0:0:root:/root:/bin/sh +EOF + +cat > /etc/group <<EOF +root:x:0:root +EOF + +cat > /etc/profile <<EOF +export PATH=/bin +EOF + +cat > /root/.profile <<EOF +export HOME=/root + +alias ll="ls -l " +alias la="ls -la " +EOF + +cat > /root/test.c <<EOF +#include <stdio.h> + +int main() { + int var = 0; + printf("Hello, world!\n"); + printf("Please input a number: \n"); + scanf("%d", &var); + if (var > 0) { + printf("You typed a positive number.\n"); + } else if (var == 0 ) { + printf("You input a zero.\n"); + } else { + printf("You typed a negative number.\n"); + } + return 0; +} +EOF + +exec sh -l + +# We don't have a working init yet, so we use busybox sh directly for now. +# exec /mnt/init /bin/sh -c 'exec sh -l < /dev/ttyS0 > /dev/ttyS0 2> /dev/ttyS0' diff --git a/user-programs/init_script_riscv64.sh b/user-programs/init_script_riscv64.sh deleted file mode 100644 index 52b2628c..00000000 --- a/user-programs/init_script_riscv64.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/mnt/busybox sh - -BUSYBOX=/mnt/busybox - -freeze() { - echo "an error occurred while executing '''$@''', freezing..." >&2 - - while true; do - true - done -} - -do_or_freeze() { - if $@; then - return - fi - - freeze $@ -} - -do_or_freeze $BUSYBOX mkdir -p /dev - -do_or_freeze $BUSYBOX mknod -m 666 /dev/console c 5 1 -do_or_freeze $BUSYBOX mknod -m 666 /dev/null c 1 3 -do_or_freeze $BUSYBOX mknod -m 666 /dev/zero c 1 5 -do_or_freeze $BUSYBOX mknod -m 666 /dev/vda b 8 0 -do_or_freeze $BUSYBOX mknod -m 666 /dev/vda1 b 8 1 -do_or_freeze $BUSYBOX mknod -m 666 /dev/vdb b 8 16 -do_or_freeze $BUSYBOX mknod -m 666 /dev/ttyS0 c 4 64 -do_or_freeze $BUSYBOX mknod -m 666 /dev/ttyS1 c 4 65 - -echo -n -e "deploying busybox... " >&2 - -do_or_freeze $BUSYBOX mkdir -p /bin -do_or_freeze $BUSYBOX --install -s /bin -do_or_freeze $BUSYBOX mkdir -p /lib - -export PATH="/bin" - -echo ok >&2 - -do_or_freeze mkdir -p /etc /root /proc -do_or_freeze mount -t procfs proc proc - -# Check if the device /dev/vdb is available and can be read -if dd if=/dev/vdb of=/dev/null bs=512 count=1; then - echo -n -e "Mounting the ext4 image... " >&2 - do_or_freeze mkdir -p /mnt1 - do_or_freeze mount -t ext4 /dev/vdb /mnt1 - echo ok >&2 -fi - -cp /mnt/ld-musl-i386.so.1 /lib/ld-musl-i386.so.1 -ln -s /lib/ld-musl-i386.so.1 /lib/libc.so - -cat > /etc/passwd <<EOF -root:x:0:0:root:/root:/mnt/busybox sh -EOF - -cat > /etc/group <<EOF -root:x:0:root -EOF - -cat > /etc/profile <<EOF -export PATH=/bin -EOF - -cat > /root/.profile <<EOF -export HOME=/root - -alias ll="ls -l " -alias la="ls -la " -EOF - -cat > /root/test.c <<EOF -#include <stdio.h> - -int main() { - int var = 0; - printf("Hello, world!\n"); - printf("Please input a number: \n"); - scanf("%d", &var); - if (var > 0) { - printf("You typed a positive number.\n"); - } else if (var == 0 ) { - printf("You input a zero.\n"); - } else { - printf("You typed a negative number.\n"); - } - return 0; -} -EOF - -exec $BUSYBOX sh -l < /dev/ttyS0 > /dev/ttyS0 2> /dev/ttyS0 - -# We don't have a working init yet, so we use busybox sh directly for now. -# exec /mnt/init /bin/sh -c 'exec sh -l < /dev/ttyS0 > /dev/ttyS0 2> /dev/ttyS0' diff --git a/user-programs/init_script_x86_64.sh b/user-programs/init_script_x86_64-acpi_compat.sh similarity index 100% rename from user-programs/init_script_x86_64.sh rename to user-programs/init_script_x86_64-acpi_compat.sh diff --git a/x86_64-unknown-none.json b/x86_64-unknown-none.json index 77e4d1dd..a66b07ba 100644 --- a/x86_64-unknown-none.json +++ b/x86_64-unknown-none.json @@ -3,8 +3,8 @@ "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", "arch": "x86_64", "target-endian": "little", - "target-pointer-width": "64", - "target-c-int-width": "32", + "target-pointer-width": 64, + "target-c-int-width": 32, "os": "none", "executables": true, "linker-flavor": "ld.lld",