From ad691cd5e9f76aa9e486d93b560c58d5c8ff88c4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 03:52:09 -0400 Subject: [PATCH 01/64] drm/asahi: publish bound G17 compute work without the admission passes A compute command on a bound queue went through every admission step on each submission. ensure_compute took the device mutex for a replay and witness pass, shared_compute took a preparation lease and the mutex again, and build_compute did both once more to find its queue already bound. compute_packet then took a third lease and the mutex a fourth time, replaying compute again before publishing. Compute also never published from its submit ioctl, so even a command with nothing to wait for went through the scheduler worker first. Its completions never fed the CPU-capacity detector, so the host loop feeding a compute-bound GPU ran at the lowest CPU performance level. Check what ensure_compute establishes under the one device-mutex section that publishes, and run the admission passes only when that check fails. Publish a dependency-free compute command from its submit ioctl when its queue is bound and nothing has to wait, as renders do. Keep a separate backlog per engine so that a command waiting in one engine's scheduler does not stop the other engine's inline publication. Report compute dispatch envelopes to the detector as render passes are. --- drivers/gpu/drm/asahi/g17/completion.rs | 8 +- drivers/gpu/drm/asahi/g17/feed.rs | 2 +- drivers/gpu/drm/asahi/g17/initdata.rs | 6 + drivers/gpu/drm/asahi/g17/job.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 5 +- drivers/gpu/drm/asahi/g17/runtime.rs | 108 +++++++++++++++++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 16 ++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 1 + .../drm/asahi/g17/runtime/render/submit.rs | 4 +- 9 files changed, 138 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 4f5b87bc853dc..dacd6de9a3a48 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -546,7 +546,13 @@ impl Completion { return; } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { - feed.note_render_pass(ticks[2], ticks[3]); + // Compute reports its dispatch envelope in the first pair, a + // render its fragment pass in the second. + if self.compute.is_some() { + feed.note_pass(ticks[0], ticks[1]); + } else { + feed.note_pass(ticks[2], ticks[3]); + } } if let Err(error) = result { // Spared work cancelled after another VM's recovery is not this client's device loss. diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 0045dff413922..760f352046192 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -410,7 +410,7 @@ impl Feed { Monotonic::ktime_get() as u64 } - pub(crate) fn note_render_pass(&self, start: u64, end: u64) { + pub(crate) fn note_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 10af261aeb3ac..049c93b6a466a 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -394,6 +394,12 @@ impl InitData { Ok(()) } + pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { + self.compute_shared + .as_ref() + .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) + } + pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { let shared = self.compute_shared.as_ref().ok_or(EIO)?; if vm.compute_pool_mappings_match(shared.id) { diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 3b47d361c2ef9..fd2cc878b534e 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -429,7 +429,7 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + if !(early && self.backend.try_early(&packet)) { self.backend.count_backlog(&packet); } job.push(); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 6a9b15789339f..1f57f2913a020 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -216,7 +216,7 @@ impl Queue { destinations, engine == Engine::Compute, host_predecessors.clone().next().is_some(), - if engine == Engine::Render { self.backend.feed() } else { None }, + self.backend.feed(), )?; let publication = completion.publication() .map(|target| PreparedPublication::new(target, host_predecessors)) @@ -298,8 +298,7 @@ impl Queue { checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - let early = engine == Engine::Render - && dependencies.is_empty() + let early = dependencies.is_empty() && self.frontiers[index].len() < EARLY_DEPTH; let packet = Packet::new( id, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6f4a5355d6eb3..284c5d7bc1eb9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -441,6 +441,7 @@ pub(super) struct Backend { pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + compute_backlog: core::sync::atomic::AtomicU32, closed: core::sync::atomic::AtomicBool, } @@ -521,6 +522,7 @@ impl Backend { context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + compute_backlog: core::sync::atomic::AtomicU32::new(0), closed: core::sync::atomic::AtomicBool::new(false), }, GFP_KERNEL, @@ -1163,6 +1165,18 @@ impl super::Firmware { } impl Registry { + /// Everything ensure_compute establishes for `owner`: the shared compute + /// objects mapped into its VM, and its queue bound to `context` with a + /// pool that fits and no spared witness to release. + fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { + let Some(queue) = self.compute(owner) else { + return Ok(false); + }; + Ok(!queue.spared_pending() + && queue.pool_fits(required) + && queue.matches(owner, queue.qid(), context)?) + } + fn release_compute_witness( &mut self, init: &super::initdata::InitData, @@ -1317,6 +1331,85 @@ impl Backend { Err(EBUSY) } + /// Publishes a dependency-free compute command from its submit ioctl when + /// its queue is already bound and nothing has to wait, as renders do. Any + /// wait leaves the command, unpublished, to its scheduler. + pub(super) fn try_compute_early(&self, packet: &Arc) -> bool { + if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = self.compute_inline(packet, &mut deferred); + if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { + deferred.finish(); + packet.abandon_publication(); + return false; + } + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } + + /// compute_packet's publication section without its waits: every condition + /// it would sleep or retry for returns FALLBACK instead. + fn compute_inline( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { + return Err(render::FALLBACK); + } + let lease = self.enter_preparation(false)?; + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => return Err(render::FALLBACK), + result => { + result?; + } + } + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() + || self.shared.crashed.load(Ordering::Acquire) + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(render::FALLBACK); + } + firmware.replay_compute(deferred)?; + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + return Err(render::FALLBACK); + } + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { + Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { + Err(render::FALLBACK) + } + result => result, + } + } + /// Called after taking this packet's publication claim. All waits and work /// state allocation precede the device mutex; only the checked epoch commits. fn compute_packet( @@ -1326,6 +1419,10 @@ impl Backend { ) -> Result { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + // A bound queue needs none of ensure_compute's admission passes. Try + // without them; the publication section below checks the same state. + let mut bound = true; let mut recovery_polls = 0; let mut state_delay = 20i64; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); @@ -1341,8 +1438,10 @@ impl Backend { // All DRM dependencies were satisfied before this scheduler callback. // Recheck on every retry: pressure may yield an idle binding between // admission and publication without changing this logical queue. - Backend::ensure_compute(self, scratch.pages(), &admission)?; - admission.check()?; + if !bound { + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + } let lease = self.shared.preparations.enter_until(&admission)?; // Work states free as this context's earlier commands retire. // Wait for one under the packet's admission deadline, as render @@ -1387,7 +1486,10 @@ impl Backend { continue; } firmware.replay_compute(deferred)?; - if firmware.queues.compute(self.owner).is_none() { + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + bound = false; drop(state); drop(lease); continue; diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7d690470cd7b5..c75b525413ee8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,7 +13,14 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog.fetch_sub(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_sub(1, Ordering::AcqRel); + } + } + + pub(super) fn backlog_of(&self, packet: &Packet) -> &core::sync::atomic::AtomicU32 { + match packet.engine() { + Engine::Render => &self.backlog, + Engine::Compute => &self.compute_backlog, } } @@ -274,11 +281,14 @@ impl job::Backend for Backend { } fn try_early(&self, packet: &Arc) -> bool { - self.try_render_early(packet) + match packet.engine() { + Engine::Render => self.try_render_early(packet), + Engine::Compute => self.try_compute_early(packet), + } } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog.fetch_add(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_add(1, Ordering::AcqRel); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 59db76b9f109a..c657a45bdfa48 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -16,6 +16,7 @@ mod settle; mod submit; pub(super) mod teardown; +pub(super) use submit::FALLBACK; use super::{DeferredBatch, Registry}; use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; use crate::hw::t8140::queues::RENDER_SLOTS; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index b77fc13b7ed2f..6c5710554e7f5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -17,13 +17,13 @@ use kernel::{ /// Internal result of a publication attempted from the submitting ioctl that /// met contention before anything became visible: the scheduler publishes the /// command instead. Never returned to userspace. -pub(super) const FALLBACK: Error = EINPROGRESS; +pub(in crate::g17::runtime) const FALLBACK: Error = EINPROGRESS; impl Backend { /// The submitting ioctl holds the logical queue's mutex and never sleeps /// for contention or a closed preparation gate: it returns FALLBACK, and /// the scheduler's worker publishes the command and owns the waits. - pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + pub(in crate::g17::runtime) fn enter_preparation(&self, sleep: bool) -> Result { if sleep { return self.shared.preparations.enter(false); } From 1121ae36b8d426be24c41ae360281b5f0709a69d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 03:53:48 -0400 Subject: [PATCH 02/64] drm/asahi: wake G17 render publication when its pair is returned A render whose pair was out on loan slept 200 us before trying again. The event worker lends each pair with completions out for an off-lock observation after every completion event. At queue depths of two and more, completions and submissions interleave, so a submission often found its pair on loan for a few microseconds and then slept the full 200 us. Count the device's change notifications. Every pair return already notifies under the device mutex. Wait for the notification under that mutex, unless one arrived since the attempt started, with the same deadline as before. --- drivers/gpu/drm/asahi/g17/mod.rs | 15 +++++++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 4 ++-- drivers/gpu/drm/asahi/g17/runtime/render/grow.rs | 2 +- .../gpu/drm/asahi/g17/runtime/render/observe.rs | 2 +- .../gpu/drm/asahi/g17/runtime/render/prepare.rs | 6 +++--- .../gpu/drm/asahi/g17/runtime/render/submit.rs | 14 ++++++++++---- 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 282ddcc5e6b0a..3e03c40442c9d 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -53,6 +53,7 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, + AtomicU64, Ordering, // }; @@ -793,6 +794,9 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, + /// Counts `changed` notifications, so that a waiter which checked under + /// the device mutex can tell whether it already missed one. + changes: AtomicU64, } impl_has_work! { @@ -882,6 +886,16 @@ impl Shared { drop(retired); } + /// Wakes the waiters for a returned pair, binding or capacity change. + fn note_changed(&self) { + self.changes.fetch_add(1, Ordering::AcqRel); + self.changed.notify_all(); + } + + fn changes(&self) -> u64 { + self.changes.load(Ordering::Acquire) + } + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { // Already queued if this fails, which is just as good. @@ -1061,6 +1075,7 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), + changes: AtomicU64::new(0), }), GFP_KERNEL, )?; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 284c5d7bc1eb9..3a496d2728f76 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -594,7 +594,7 @@ impl Backend { .queues .compute_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); drop(lease); if result != Err(EAGAIN) { return result; @@ -654,7 +654,7 @@ impl Backend { .queues .compute_scheduler_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); result } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index e3cb174721aaa..20569af84bc7b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -280,7 +280,7 @@ impl crate::g17::Shared { return Err(EIO); } firmware.queues.render.grow.active = None; - self.changed.notify_all(); + self.note_changed(); if !lease.is_current() || request.halt_count != firmware.recovery.generation() { continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 4ede20d316c64..52bdd964ca4ae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -164,7 +164,7 @@ impl crate::g17::Shared { return Err(error); } let firmware = (*state).as_deref_mut().ok_or(EIO)?; - self.changed.notify_all(); + self.note_changed(); if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index e2034c3176595..04afe2e05c107 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -426,7 +426,7 @@ impl super::super::Backend { } firmware.render_grow_ready() }; - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -548,7 +548,7 @@ impl super::super::Backend { .release(id)?; } } - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -711,7 +711,7 @@ impl super::super::Backend { .ok_or(EIO)? .release_owner(); } - self.shared.changed.notify_all(); + self.shared.note_changed(); let published = (|| { let entry = firmware.queues.render.entry(slot)?; if entry.pending_error.is_some() || entry.pending_timeout { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 6c5710554e7f5..e7d0cf1168c98 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -144,7 +144,7 @@ impl Backend { /// scheduler it last ran for to be consumed and the pair parked. The event /// worker observes both, so kick it on every poll; time out only after the /// same bound as a capacity wait. - fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { + fn wait_render_busy(&self, since: &mut Option, seen: u64, sleep: bool) -> Result { if !sleep { return Err(FALLBACK); } @@ -153,7 +153,12 @@ impl Backend { return Err(ETIMEDOUT); } self.shared.queue_events(); - fsleep(Delta::from_micros(200)); + // Every pair return notifies under the device mutex: wake on it rather + // than sleeping out a fixed interval behind a short observation loan. + let mut state = self.shared.state.lock(); + if self.shared.changes() == seen { + let _ = self.shared.changed.wait_timeout(&mut state, kernel::time::msecs_to_jiffies(1)); + } Ok(()) } @@ -194,6 +199,7 @@ impl Backend { let mut recovery_polls = 0; let capacity_started = super::super::now_ns(); loop { + let seen = self.shared.changes(); if packet.cancel_requested() { return Err(ECANCELED); } @@ -242,7 +248,7 @@ impl Backend { } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy_since, seen, sleep)?; continue; } result => result?, @@ -270,7 +276,7 @@ impl Backend { self.shared.queue_events(); fsleep(Delta::from_millis(1)); } else if error == EBUSY { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy_since, seen, sleep)?; } else if !sleep { return Err(FALLBACK); } From abd85935bd4db7db6be23b8194b1173c9a8988c5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 11:05:39 -0400 Subject: [PATCH 03/64] drm/asahi: clear each G17 compute descriptor only once Object::initialize already clears the complete descriptor before calling its encoder. Fill that zeroed representation directly instead of clearing the same 16 KiB again. Name the encoder write_zeroed to make its precondition explicit. The firmware bytes and validation remain unchanged. The matched module passed the targeted compute/synchronization gate and checked GPU readbacks. --- drivers/gpu/drm/asahi/g17/fw/compute.rs | 8 ++++---- drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0cb7304e30677..b1afdad816b6b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -12,7 +12,7 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; use super::queue::CommandTag; use super::{ - clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, }; @@ -507,8 +507,9 @@ impl ComputeDescriptor { const UNK_FBA: u32 = 0xe0a0_0001; const UNK_FC5: u8 = 0x9f; - /// Writes the complete descriptor. - pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { + /// Fills a descriptor cleared by `Object::initialize`. The unused firmware + /// fields and padding retain that zero initialization. + pub(crate) fn write_zeroed(&mut self, args: &ComputeArgs) -> Result { if args.cdm_end_va < args.cdm_va || (args.sampler_heap_va == 0) != (args.sampler_count == 0) || args.sampler_heap_va & 7 != 0 @@ -522,7 +523,6 @@ impl ComputeDescriptor { let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; let sampler_max = sampler_max(args.sampler_count)?; - clear(self); self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 713809907dd4b..b75e00da0dad4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1904,7 +1904,7 @@ impl Queue { .object .initialize::( kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), + |descriptor| descriptor.write_zeroed(&args), )?; if first { self.graph.context_high.write( From 3ef3720890f8c085f18e4f5e35db6853075b121c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 11:05:53 -0400 Subject: [PATCH 04/64] drm/asahi: retain compute pool backing through pending channel admission A publisher waiting for a physical compute channel reclaimed idle USC backing even though freeing backing cannot create another QID. The normal warm-pool limit could also evict a retired channel while another runnable publisher was waiting to reuse it. Subsequent handoffs repeatedly allocated and cleared fresh pool backing. Distinguish channel pressure from backing allocation pressure. Track unbound compute admission through publication, cancellation and error, and retain backing for the existing compute channels while that demand remains. The last publisher schedules normal idle reclamation. Allocation-pressure reclamation, the idle warm-pool limit and expiry remain in force. Expose the admission count in the capacity diagnostics. On Neo, the matched 16-client probe drops pool allocations from 4888 to 53 and median submission time from 257 to 109 us. The Asahi KUnit suites pass 67/67; the targeted compute/synchronization gate passes 116 with 18 skips. Full mustpass qualification remains pending. --- drivers/gpu/drm/asahi/g17/capacity.rs | 1 + drivers/gpu/drm/asahi/g17/mod.rs | 4 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 57 ++++++++++++++++++++++----- 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index bcbbb2e7e875d..8a7f48d8a3f35 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -55,6 +55,7 @@ impl kernel::debugfs::Writer for View { let state = self.0.state.lock(); let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; writeln!(f, "online=1")?; + writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; firmware.queues.write_capacity(f)?; firmware.pool.write_capacity(f)?; writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 3e03c40442c9d..6b9e9903f3a0d 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -53,6 +53,7 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, + AtomicU32, AtomicU64, Ordering, // }; @@ -772,6 +773,8 @@ struct Shared { #[pin] feed_work: Work, preparations: Arc, + /// Unbound compute publishers which can reuse a retired channel. + compute_demand: AtomicU32, #[pin] work: Work, #[pin] @@ -1066,6 +1069,7 @@ impl Gpu { feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 3a496d2728f76..beffae857a034 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -34,6 +34,30 @@ const FIRST_COMPUTE_QID: u8 = 4; const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; +/// Pending admission keeps exactly retired compute backing available for reuse. +/// The guard spans retries and disappears on publication, cancellation or error. +/// It borrows Shared, so the accounting cannot retain the device after teardown. +struct ComputeDemand<'a>(&'a Arc); + +impl<'a> ComputeDemand<'a> { + fn new(shared: &'a Arc) -> Self { + // Each admitted logical queue has at most one scheduler publisher. + // The execution-context bound is far below the counter's range. + shared.compute_demand.fetch_add(1, Ordering::AcqRel); + Self(shared) + } +} + +impl Drop for ComputeDemand<'_> { + fn drop(&mut self) { + if self.0.compute_demand.fetch_sub(1, Ordering::AcqRel) == 1 { + // Reinstate normal idle retention even if the last publisher was + // cancelled before a kick could produce another event. + self.0.queue_events(); + } + } +} + /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. #[derive(Clone, Copy)] @@ -782,7 +806,7 @@ impl Backend { Err(error) if error == ENOSPC => { drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_compute_capacity(admission, true)?; continue; } result => result?, @@ -795,11 +819,12 @@ impl Backend { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, Err(error) if error == EAGAIN => { - // The event worker owns release observation and - // drops vacated aliases/backing off this mutex. + // All physical channels are occupied. Retain their + // USC backing for exact-retirement handoff; releasing + // it cannot create a QID and forces fresh allocation. drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_compute_capacity(admission, false)?; continue; } Err(error) if error == ENOSPC => { @@ -833,7 +858,7 @@ impl Backend { let (pages, ring) = match built { Err(error) if error == ENOSPC => { drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_compute_capacity(admission, true)?; continue; } result => result?, @@ -922,7 +947,7 @@ impl Backend { } drop(lease); if result == Err(ENOSPC) { - self.wait_compute_capacity(admission)?; + self.wait_compute_capacity(admission, true)?; continue; } if result != Err(EAGAIN) { return result; } @@ -1423,6 +1448,7 @@ impl Backend { // A bound queue needs none of ensure_compute's admission passes. Try // without them; the publication section below checks the same state. let mut bound = true; + let mut demand = None; let mut recovery_polls = 0; let mut state_delay = 20i64; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); @@ -1439,6 +1465,7 @@ impl Backend { // Recheck on every retry: pressure may yield an idle binding between // admission and publication without changing this logical queue. if !bound { + demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); Backend::ensure_compute(self, scratch.pages(), &admission)?; admission.check()?; } @@ -1670,6 +1697,11 @@ impl super::Firmware { // retained compute pool stays warm: render and new compute need it more. let cap = if super::freelist::lower_pressure() { 0 + } else if self.primary.state.shared.compute_demand.load(Ordering::Acquire) != 0 { + // A retired channel being requested by another runnable publisher + // is not an unused warm pool. Keep its existing backing through + // handoff; real allocation pressure above still forces reclamation. + super::kick::COMPUTE_CHANNELS } else { COMPUTE_WARM_CAP }; @@ -1714,10 +1746,10 @@ impl Backend { pub(super) fn wait_capacity(&self, started: u64) -> Result { let check = || self.admission_open(); let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); - self.wait_compute_capacity(&admission) + self.wait_compute_capacity(&admission, true) } - fn wait_compute_capacity(&self, admission: &Admission<'_>) -> Result { + fn wait_compute_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { admission.check()?; let mut deferred = DeferredBatch::publication()?; let result: Result = (|| { @@ -1738,8 +1770,13 @@ impl Backend { } // Consumption, not publication or an elapsed timeout, frees backing. firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; - firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; - if !firmware.render_control_backpressured() { + // Pool eviction addresses backing allocation pressure, not channel + // occupancy. Normal idle expiry and the retained-pool cap still run + // in the event worker for either kind of admission wait. + if reclaim_backing { + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + } + if reclaim_backing && !firmware.render_control_backpressured() { let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } } From f35d7ea1b37395a9bc1241d0fafcb698e0c12d46 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 11:11:30 -0400 Subject: [PATCH 05/64] dma-buf: stop polling an array after its first pending member Fence arrays require every member to signal. During syncobj timeline walks, a pending predecessor already proves that the array is incomplete, yet the lazy signaled callback polls every successor too. Return false immediately at that predecessor, keeping callback accounting and the enabled-signaling path unchanged. Add checks for out-of-order completion, avoiding successor polls, and callback installation after a partial lazy poll. Remove the obsolete signal-on-any description from the constructor documentation. --- drivers/dma-buf/dma-fence-array.c | 29 ++++++----- drivers/dma-buf/st-dma-fence-unwrap.c | 75 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c index 5e10e8df372f1..64abdcc430580 100644 --- a/drivers/dma-buf/dma-fence-array.c +++ b/drivers/dma-buf/dma-fence-array.c @@ -107,17 +107,11 @@ static bool dma_fence_array_signaled(struct dma_fence *fence) unsigned int i; /* - * We need to read num_pending before checking the enable_signal bit - * to avoid racing with the enable_signaling() implementation, which - * might decrement the counter, and cause a partial check. - * atomic_read_acquire() pairs with atomic_dec_and_test() in - * dma_fence_array_enable_signaling() - * - * The !--num_pending check is here to account for the any_signaled case - * if we race with enable_signaling(), that means the !num_pending check - * in the is_signalling_enabled branch might be outdated (num_pending - * might have been decremented), but that's fine. The user will get the - * right value when testing again later. + * Read the counter before the enable bit so callback installation + * cannot make a partial count look complete. The acquire pairs with + * the counter decrements in dma_fence_array_enable_signaling(). + * If enabling races with the lazy scan, every child must still be + * observed complete before that scan can return true. */ num_pending = atomic_read_acquire(&array->num_pending); if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &array->base.flags)) { @@ -126,8 +120,16 @@ static bool dma_fence_array_signaled(struct dma_fence *fence) return false; } + /* + * Arrays require every member to complete. A pending member already + * proves the array incomplete; polling its successors only repeats work + * during timeline-chain walks. Keep the completion counter and the + * enabled-signaling path unchanged. + */ for (i = 0; i < array->num_fences; ++i) { - if (dma_fence_is_signaled(array->fences[i]) && !--num_pending) + if (!dma_fence_is_signaled(array->fences[i])) + return false; + if (!--num_pending) goto signal; } return false; @@ -256,8 +258,7 @@ EXPORT_SYMBOL(dma_fence_array_init); * and fill it with the fences it wants to add to the object. Ownership of this * array is taken and dma_fence_put() is used on each fence on release. * - * If @signal_on_any is true the fence array signals if any fence in the array - * signals, otherwise it signals when all fences in the array signal. + * The array signals when every fence in the array signals. */ struct dma_fence_array *dma_fence_array_create(int num_fences, struct dma_fence **fences, diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c index 4d9d313b460cb..17761f8de04a7 100644 --- a/drivers/dma-buf/st-dma-fence-unwrap.c +++ b/drivers/dma-buf/st-dma-fence-unwrap.c @@ -15,6 +15,7 @@ struct mock_fence { struct dma_fence base; spinlock_t lock; + atomic_t polls; }; static const char *mock_name(struct dma_fence *f) @@ -22,9 +23,18 @@ static const char *mock_name(struct dma_fence *f) return "mock"; } +static bool mock_signaled(struct dma_fence *fence) +{ + struct mock_fence *f = container_of(fence, struct mock_fence, base); + + atomic_inc(&f->polls); + return false; +} + static const struct dma_fence_ops mock_ops = { .get_driver_name = mock_name, .get_timeline_name = mock_name, + .signaled = mock_signaled, }; static struct dma_fence *__mock_fence(u64 context, u64 seqno) @@ -36,6 +46,7 @@ static struct dma_fence *__mock_fence(u64 context, u64 seqno) return NULL; spin_lock_init(&f->lock); + atomic_set(&f->polls, 0); dma_fence_init(&f->base, &mock_ops, &f->lock, context, seqno); return &f->base; @@ -584,7 +595,71 @@ static void test_unwrap_merge_complex_seqno(struct kunit *test) dma_fence_put(f1); } +/* + * A pending all-of array needs no polls of later members. Keep its + * independent member fences so their polling and out-of-order completion + * remain observable without enabling callbacks on the array. + */ +static void test_array_pending_prefix(struct kunit *test) +{ + struct dma_fence *first = mock_fence(), *last = mock_fence(); + struct dma_fence *array; + struct mock_fence *last_mock; + + if (!first || !last) { + KUNIT_FAIL(test, "Could not allocate member fences"); + goto out; + } + last_mock = container_of(last, struct mock_fence, base); + array = mock_array(2, dma_fence_get(first), dma_fence_get(last)); + if (!array) { + KUNIT_FAIL(test, "Could not allocate fence array"); + goto out; + } + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + KUNIT_EXPECT_EQ(test, atomic_read(&last_mock->polls), 0); + dma_fence_signal(last); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + dma_fence_signal(first); + KUNIT_EXPECT_TRUE(test, dma_fence_is_signaled(array)); + dma_fence_put(array); +out: + dma_fence_put(last); + dma_fence_put(first); +} + +static void test_array_pending_suffix(struct kunit *test) +{ + struct dma_fence *first = mock_fence(), *last = mock_fence(); + struct dma_fence *array; + + if (!first || !last) { + KUNIT_FAIL(test, "Could not allocate member fences"); + goto out; + } + array = mock_array(2, dma_fence_get(first), dma_fence_get(last)); + if (!array) { + KUNIT_FAIL(test, "Could not allocate fence array"); + goto out; + } + dma_fence_signal(first); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + /* Installing callbacks after a lazy partial poll must still account + * for the already signaled first member and wait for the last. + */ + dma_fence_enable_signaling(array); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + dma_fence_signal(last); + KUNIT_EXPECT_GT(test, dma_fence_wait_timeout(array, false, HZ), 0L); + dma_fence_put(array); +out: + dma_fence_put(last); + dma_fence_put(first); +} + static struct kunit_case dma_fence_unwrap_cases[] = { + KUNIT_CASE(test_array_pending_prefix), + KUNIT_CASE(test_array_pending_suffix), KUNIT_CASE(test_sanitycheck), KUNIT_CASE(test_unwrap_array), KUNIT_CASE(test_unwrap_chain), From 319805fed41bb181db8e90378beaf905f38f6297 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 14:38:24 -0400 Subject: [PATCH 06/64] drm/asahi: avoid unbound compute inline attempts on compute-only queues Compute-only contexts that yield their physical channel still take the device mutex in the inline submit path before falling back to admission. Track an advisory binding hint to skip that attempt. Actual publication retains the locked VM, epoch and binding checks, and scheduler admission ordering is unchanged. Mark contexts with validated rendering before any batch member is queued, so mixed-engine queues retain the original inline path. Validated with 69 Asahi KUnit cases, targeted compute/sync CTS (116 pass, 18 skip), mixed device churn and 16-client concurrent GPU readbacks. Matched 16-process quick submission time improves from 117.27 to 89.55 us; paired render follow-ups remain at control performance. --- drivers/gpu/drm/asahi/g17/context.rs | 19 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 5 +++++ drivers/gpu/drm/asahi/g17/queue/mod.rs | 3 +++ drivers/gpu/drm/asahi/g17/runtime.rs | 3 +++ 4 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index c93ab25d55ce3..445b282dcf5b6 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -114,6 +114,11 @@ pub(crate) struct Context { scheduler: Arc, scheduler_generation: u32, published: AtomicBool, + /// Advisory only: publication still checks the physical binding under the + /// device mutex. False skips an inline attempt known to need admission. + compute_binding_hint: AtomicBool, + /// Mixed-engine contexts keep the original inline publication path. + render_seen: AtomicBool, qos: QosLease, owner_pid: u32, states: WorkStates, @@ -189,6 +194,8 @@ impl Context { scheduler, scheduler_generation, published: AtomicBool::new(false), + compute_binding_hint: AtomicBool::new(false), + render_seen: AtomicBool::new(false), qos, owner_pid, states: WorkStates::new(), @@ -203,6 +210,18 @@ impl Context { pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_binding_hint(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + || self.compute_binding_hint.load(Ordering::Relaxed) + } + pub(crate) fn note_render(&self) { + if !self.render_seen.load(Ordering::Relaxed) { + self.render_seen.store(true, Ordering::Release); + } + } + pub(crate) fn note_compute_binding(&self, owned: bool) { + self.compute_binding_hint.store(owned, Ordering::Relaxed); + } pub(crate) fn compute_timestamps(&self) -> &Arc { &self.compute_timestamps } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b75e00da0dad4..d2ec8da9c6082 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -557,6 +557,7 @@ impl Queue { let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); + context.note_compute_binding(true); Ok(Self { owner: Some(owner), retained_owner: None, @@ -1274,6 +1275,7 @@ impl Queue { self.retained_owner = None; } context.mark_published(); + context.note_compute_binding(true); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -1284,6 +1286,9 @@ impl Queue { /// True allows the registry to cancel the never-published QID and drop this graph. An /// installed graph remains retained even after the logical owner exits without work. pub(crate) fn release_owner(&mut self) -> Result { + if let Some(binding) = &self.binding { + binding.context.note_compute_binding(false); + } if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { return Ok(true); } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 1f57f2913a020..96e566d3971ff 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -409,6 +409,9 @@ impl crate::queue::Queue for Queue { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if pending.iter().any(|(command, _, _)| matches!(command, Validated::Render { .. })) { + self.context.note_render(); + } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index beffae857a034..dbae7ea2d4021 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1411,6 +1411,9 @@ impl Backend { result?; } } + if !self.context.compute_binding_hint() { + return Err(render::FALLBACK); + } let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() From 5a4822c6fb3d1fc0a83d9b754b477bf1db35d782 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 15:05:59 -0400 Subject: [PATCH 07/64] drm/asahi: preserve low-pressure compute inline probes Only bypass an unbound compute-only inline attempt when multiple publishers are waiting for admission. A lone admission retains the original probe and its latency behavior. Actual publication still uses the existing locked state checks. Matched quick controls retain queue signaling latency (174.1 to 174.5 us) while 16-process submission time improves from 117.3 to 90.7 us. All 20 performance rows complete successfully. Checked mixed/concurrent GPU readbacks, 69 Asahi KUnit cases and targeted compute/sync CTS also pass. --- drivers/gpu/drm/asahi/g17/runtime.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index dbae7ea2d4021..bd95148e9eaf0 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1411,7 +1411,11 @@ impl Backend { result?; } } - if !self.context.compute_binding_hint() { + // A lone admission keeps the original latency-sensitive probe. + // Competing publishers otherwise spin on a binding they do not own. + if !self.context.compute_binding_hint() + && self.shared.compute_demand.load(Ordering::Relaxed) > 1 + { return Err(render::FALLBACK); } let mut state = self.shared.state.lock(); From 026065c435260844e39d05a182e99b128c5f9b2d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 20:29:12 -0400 Subject: [PATCH 08/64] drm/asahi: submit same-VM compute input waits through firmware Recognize eligible singleton compute producers through their exported aggregate fences. Order queued consumers after actual producer publication using the existing DRM scheduled fence, then validate the current physical ticket and emit firmware waits. Preserve CPU waits for unsupported dependencies and host outputs, and revalidate recovery epochs before acquiring consumer capacity. Retain source binding loans through consumer retirement, propagate source errors before dependent completion, and release failed submission pins when the last runtime owner retires even if exported fence references survive. Keep bounded member counters in byte atomics so submission state retains its original allocation class. Validated with 78 KUnit tests, the compute/synchronization CTS gate, chained readback, process exits and mixed compute/render pressure. Fixed-core ordinary compute matches the control while the queue-hop path improves. Existing intermittent unrelated-client loss during forced recovery remains unresolved. --- drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/g17/capacity.rs | 4 + drivers/gpu/drm/asahi/g17/completion.rs | 32 +- drivers/gpu/drm/asahi/g17/context.rs | 1 + drivers/gpu/drm/asahi/g17/fence.rs | 366 +++++++++++++------ drivers/gpu/drm/asahi/g17/job.rs | 8 + drivers/gpu/drm/asahi/g17/mod.rs | 1 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 112 +++++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 35 +- drivers/gpu/drm/asahi/g17/runtime.rs | 127 ++++++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 49 +++ drivers/gpu/drm/asahi/g17/sync_input.c | 42 +++ drivers/gpu/drm/asahi/g17/sync_input.h | 18 + drivers/gpu/drm/asahi/g17/sync_input.rs | 245 +++++++++++++ 14 files changed, 905 insertions(+), 137 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/sync_input.c create mode 100644 drivers/gpu/drm/asahi/g17/sync_input.h create mode 100644 drivers/gpu/drm/asahi/g17/sync_input.rs diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index bc0daf03a04d1..8f8f679532d2b 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o +asahi-y := asahi_core.o sysfs.o g17/sync_input.o asahi-$(CONFIG_DRM_ASAHI_G16) += g16/glue.o diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index 8a7f48d8a3f35..57406e5487a37 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -55,6 +55,10 @@ impl kernel::debugfs::Writer for View { let state = self.0.state.lock(); let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; writeln!(f, "online=1")?; + let (converted, published) = super::sync_input::counters(); + writeln!(f, "fw_inputs converted={} published={} native={} pending={}", + converted, published, super::sync_input::native_count(), + super::sync_input::pending_count())?; writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; firmware.queues.write_capacity(f)?; firmware.pool.write_capacity(f)?; diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index dacd6de9a3a48..406efc0e221fe 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -92,6 +92,8 @@ impl Destinations { Ok(Self(retained)) } + pub(crate) fn has_outputs(&self) -> bool { self.0.is_some() } + fn iter(&self) -> core::slice::Iter<'_, Option> { let destinations: &[Option] = self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); @@ -254,6 +256,7 @@ pub(crate) struct Completion { terminal: Option>, compute: Option, feed: Option>, + sources: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -286,6 +289,7 @@ impl Completion { compute: bool, host_predecessors: bool, feed: Option>, + sources: Option>, ) -> Result> { let fence = fence::independent(contexts)?; let ordered = !compute @@ -313,6 +317,7 @@ impl Completion { terminal, compute, feed, + sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -357,6 +362,26 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } + pub(super) fn sources(&self) -> Option<&Arc> { + self.sources.as_ref() + } + pub(super) fn is_input_source(&self, source: &Arc) -> bool { + self.output.output().member.is_source(source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { + self.output.output().member.set_source_ready(ready); + } + pub(super) fn needs_source_ready(&self) -> bool { + self.output.output().member.needs_source_ready() + } + pub(super) fn note_input_error(&self, error: Error) -> bool { + self.output.output().member.note_error(error) + } + pub(super) fn check_source_inputs(&self) -> Result { + if let Some(sources) = &self.sources { sources.check()?; } + Ok(()) + } + pub(crate) fn work_state(&self) -> Result> { self.output.output().member.work_state() } @@ -525,6 +550,7 @@ impl Completion { /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { + if let Some(sources) = &self.sources { sources.release(); } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -534,6 +560,7 @@ impl Completion { } fn release_healthy(&self) { + if let Some(sources) = &self.sources { sources.release(); } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -541,7 +568,10 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { + if result.is_ok() { + if let Err(error) = self.check_source_inputs() { result = Err(error); } + } if self.signalled.swap(true, Ordering::AcqRel) { return; } diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 445b282dcf5b6..1818fd8c6922d 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -354,6 +354,7 @@ pub(crate) struct WorkStateLease { word: AtomicU32, } impl WorkStateLease { + pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { Self { context, diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 3414e36db9d36..25e5f6b04d558 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -7,10 +7,10 @@ use super::context::{Context, WorkStateLease}; use crate::file::SyncItem; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicUsize, Ordering}; use kernel::{ bindings, c_str, - dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UniqueFence, UserFence}, new_mutex, prelude::*, str::CStr, @@ -48,106 +48,263 @@ pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { Ok(Fence::from_fence(&fence)) } -/// All command members of an ioctl share its work-state node and aggregate fence. +/// The existing aggregate-fence allocation owns the existing submission state. +/// Submission never owns this fence, so exported references cannot form a cycle. +pub(crate) struct ExportFence { + source: Option>, +} +static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); + +#[vtable] +impl FenceOps for ExportFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("g17-aggregate") + } +} + +#[inline(never)] +fn export_fallback(contexts: &FenceContexts) -> Result> { + let fence = contexts.new_fence(0, ExportFence { source: None })?; + // SAFETY: Unique, initialized and still private. Only this constructor + // publishes ExportFence objects; its payload is immutable after sharing. + let ops = unsafe { + (*fence.raw()).context = bindings::dma_fence_context_alloc(1); + (*fence.raw()).ops as usize + }; + let previous = EXPORT_OPS.load(Ordering::Acquire); + if previous == 0 { + match EXPORT_OPS.compare_exchange(0, ops, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => {}, + Err(previous) if previous == ops => {}, + Err(_) => return Err(EIO), + } + } else if previous != ops { + return Err(EIO); + } + Ok(fence) +} + +/// Caller retains the leaf fence throughout lookup. Exact registered operations +/// identify the repr(C) FenceObject payload, not a driver or timeline name. +pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) + -> Option> +{ + let expected = EXPORT_OPS.load(Ordering::Acquire); + if expected == 0 { return None; } + // SAFETY: The caller owns a live fence reference. ExportFence has a unique + // timeline callback and nontrivial payload drop. No cast precedes the exact + // registered vtable check. FenceObject begins with its dma_fence. + let object = unsafe { + if (*raw).ops as usize != expected { return None; } + &*(raw as *const FenceObject) + }; + object.source.as_ref().filter(|source| source.firmware_source).cloned() +} + +struct State { + work: Option>, + ready: Option, +} + +// One Member per parsed command, plus the enqueue guard. Exported references +// retain the allocation but do not contribute to either of these counters. +kernel::static_assert!(crate::file::MAX_COMMANDS_PER_SUBMISSION < u8::MAX as u32); + /// The initial member keeps the aggregate unsignalled until enqueue finishes. #[pin_data] pub(crate) struct Submission { - fence: UserFence, - /// Successful retirement drops only this bookkeeping owner; runtime owners - /// and failed submissions keep their work-state allocation. #[pin] - work_state: Mutex>>, - remaining: AtomicU32, + state: Mutex, + remaining: AtomicU8, error: AtomicI32, + /// Live Member objects, plus the enqueue guard, independently of signalling. + owners: AtomicU8, + firmware_source: bool, } impl Submission { - fn new(fence: UserFence, context: Arc) -> Result> { - let work_state = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; - Arc::pin_init( - pin_init!(Self { - fence, - work_state <- new_mutex!(Some(work_state), "G17 submission work state"), - remaining: AtomicU32::new(1), - error: AtomicI32::new(0), - }), - GFP_KERNEL, - ) - } - - pub(crate) fn fence(&self) -> Fence { - Fence::from_fence(&self.fence) - } - - /// Add one command before enqueue ends. The parser bounds the count to 64. - pub(crate) fn member(self: &Arc) -> Member { - self.remaining.fetch_add(1, Ordering::Relaxed); - Member { - submission: self.clone(), - complete: AtomicBool::new(false), - } + fn new(context: Arc, firmware_source: bool) -> Result> { + let work = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + Self::with_work(Some(work), firmware_source) } - - fn member_done(&self, result: Result) { - if let Err(error) = result { - let _ = self.error.compare_exchange( - 0, - error.to_errno(), - Ordering::AcqRel, - Ordering::Acquire, - ); - } + fn with_work(work: Option>, firmware_source: bool) -> Result> { + Arc::pin_init(pin_init!(Self { + state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), + remaining: AtomicU8::new(1), + error: AtomicI32::new(0), + owners: AtomicU8::new(1), + firmware_source, + }), GFP_KERNEL) + } + pub(super) fn source_status(&self) -> Result { + let complete = self.remaining.load(Ordering::Acquire) == 0; + let error = self.error.load(Ordering::Acquire); + if error != 0 { return Err(super::dependency::inherited_error(Error::from_errno(error))); } + Ok(complete) + } + pub(super) fn same_vm(&self, status: &Arc) -> bool { + self.state.lock().work.as_ref().is_some_and(|work| Arc::ptr_eq(work.status(), status)) + } + pub(super) fn ready(&self) -> Option { self.state.lock().ready.clone() } + fn set_ready(&self, ready: Fence) { + if !self.firmware_source { return; } + let mut state = self.state.lock(); + if state.work.is_some() && state.ready.is_none() { state.ready = Some(ready); } + } + fn note_error(&self, error: Error) -> bool { + self.error.compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire).is_ok() + } + fn release_state(&self) { + let old = { + let mut state = self.state.lock(); + (state.work.take(), state.ready.take()) + }; + drop(old); + } + fn owner_done(&self) { + // Every hardware owner retains its Completion/Member through exact + // retirement or stop. Exported fence FDs must not retain failed pins. + if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { self.release_state(); } + } + fn member_done(&self, fence: &UserFence, result: Result) { + if let Err(error) = result { self.note_error(error); } if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { let error = self.error.load(Ordering::Acquire); - if error != 0 { - self.fence.set_error(Error::from_errno(error)); - } else { - // Scheduler cleanup can wait behind submission on the same worker. - // Once every member and enqueue have succeeded, retained packets - // must not prevent new submissions from acquiring work-state nodes. - let retired = self.work_state.lock().take(); - drop(retired); - } - self.fence.signal(); + if error != 0 { fence.set_error(Error::from_errno(error)); } + else { self.release_state(); } + fence.signal(); } } +} +#[derive(Clone)] +pub(crate) struct Aggregate(UserFence); +impl Aggregate { + fn source(&self) -> &Arc { + // Only Outputs::publish constructs Aggregate, after setting the payload. + self.0.source.as_ref().expect("published aggregate has submission state") + } + fn fence(&self) -> Fence { Fence::from_fence(&self.0) } + pub(crate) fn member(&self) -> Member { + self.source().remaining.fetch_add(1, Ordering::Relaxed); + self.source().owners.fetch_add(1, Ordering::Relaxed); + Member { aggregate: self.clone(), complete: AtomicBool::new(false) } + } fn finish_enqueue(&self, result: Result) { - // A dropped, never-enqueued member reports cancellation. The ioctl's actual - // failure takes precedence, while published members still retain their work. - if let Err(error) = result { - self.error.store(error.to_errno(), Ordering::Release); - } - self.member_done(result); + if let Err(error) = result { self.source().error.store(error.to_errno(), Ordering::Release); } + self.source().member_done(&self.0, result); + self.source().owner_done(); } } -/// One command's contribution to its submission, including construction failure. pub(crate) struct Member { - submission: Arc, + aggregate: Aggregate, complete: AtomicBool, } - impl Member { + pub(super) fn needs_source_ready(&self) -> bool { self.aggregate.source().firmware_source } pub(crate) fn work_state(&self) -> Result> { - self.submission - .work_state - .lock() - .as_ref() - .cloned() - .ok_or(ECANCELED) + self.aggregate.source().state.lock().work.clone().ok_or(ECANCELED) } - + pub(super) fn is_source(&self, source: &Arc) -> bool { + self.aggregate.source().firmware_source && Arc::ptr_eq(self.aggregate.source(), source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { self.aggregate.source().set_ready(ready); } + pub(super) fn note_error(&self, error: Error) -> bool { self.aggregate.source().note_error(error) } pub(crate) fn complete(&self, result: Result) { if !self.complete.swap(true, Ordering::AcqRel) { - self.submission.member_done(result); + self.aggregate.source().member_done(&self.aggregate.0, result); } } } +#[kunit_tests(asahi_g17_aggregate_promise)] +mod tests { + use super::*; + use kernel::sync::LockClassKey; + static KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + + fn contexts() -> FenceContexts { + FenceContexts::new(1, c_str!("asahi_promise_test"), KEY).unwrap() + } + fn aggregate(contexts: &FenceContexts) -> Aggregate { + let mut fence = export_fallback(contexts).unwrap(); + fence.source = Some(Submission::with_work(None, true).unwrap()); + Aggregate(fence.into()) + } + + #[test] + fn exact_export_identity_recognizes_unpublished_source() { + let contexts = contexts(); + let aggregate = aggregate(&contexts); + let plain = independent(&contexts).unwrap(); + // SAFETY: Both references remain owned throughout the checked lookup. + assert!(unsafe { exported_source_raw(plain.raw()) }.is_none()); + let source = unsafe { exported_source_raw(aggregate.0.raw()) }.unwrap(); + assert!(Arc::ptr_eq(&source, aggregate.source())); + assert_eq!(source.source_status().unwrap(), false); + aggregate.finish_enqueue(Ok(())); + assert_eq!(source.source_status().unwrap(), true); + } + + #[test] + fn failed_export_releases_state_after_last_runtime_owner() { + let contexts = contexts(); + let aggregate = aggregate(&contexts); + let source = aggregate.source().clone(); + source.state.lock().ready = Some(Fence::from_fence(&independent(&contexts).unwrap())); + let member = aggregate.member(); + aggregate.finish_enqueue(Ok(())); + member.complete(Err(EIO)); + assert_eq!(super::super::job::fence_status(&aggregate.fence()), EIO.to_errno()); + assert!(source.state.lock().ready.is_some()); + drop(member); + // Keep both the exported fence and source state alive, as userspace can. + assert!(source.state.lock().ready.is_none()); + assert_eq!(source.owners.load(Ordering::Acquire), 0); + } + + #[test] + fn actual_enqueue_error_wins_over_dropped_member_cancellation() { + let aggregate = aggregate(&contexts()); + drop(aggregate.member()); + aggregate.finish_enqueue(Err(ENOMEM)); + assert_eq!(super::super::job::fence_status(&aggregate.fence()), ENOMEM.to_errno()); + } + + #[test] + fn rejected_private_fence_never_exports_source_state() { + let fence = export_fallback(&contexts()).unwrap(); + fence.set_error(ENOMEM); + fence.signal(); + // SAFETY: The unique fence owns its initialized allocation. + assert!(unsafe { exported_source_raw(fence.raw()) }.is_none()); + } + + #[test] + fn maximum_submission_keeps_all_members_and_enqueue_guard() { + let aggregate = aggregate(&contexts()); + let source = aggregate.source().clone(); + let mut members: [Option; crate::file::MAX_COMMANDS_PER_SUBMISSION as usize] = + core::array::from_fn(|_| Some(aggregate.member())); + assert_eq!(source.remaining.load(Ordering::Acquire), 65); + assert_eq!(source.owners.load(Ordering::Acquire), 65); + aggregate.finish_enqueue(Ok(())); + for member in &members { + assert_eq!(super::super::job::fence_status(&aggregate.fence()), 0); + member.as_ref().unwrap().complete(Ok(())); + } + assert!(super::super::job::fence_status(&aggregate.fence()) > 0); + assert_eq!(source.remaining.load(Ordering::Acquire), 0); + for member in &mut members { drop(member.take()); } + assert_eq!(source.owners.load(Ordering::Acquire), 0); + } +} impl Drop for Member { fn drop(&mut self) { self.complete(Err(ECANCELED)); + self.aggregate.source().owner_done(); } } @@ -155,83 +312,54 @@ fn install(outputs: impl Iterator, fence: &Fence) { for mut sync in outputs { if let Some(chain) = sync.chain_fence.take() { sync.syncobj.add_point(chain, fence, sync.timeline_value); - } else { - sync.syncobj.replace_fence(Some(fence)); - } + } else { sync.syncobj.replace_fence(Some(fence)); } } } -/// Keeps resolved output syncobjs reachable on every fallible submission path. -/// Input syncobjs stay in the same vector, avoiding a separate output allocation. pub(crate) struct Outputs { - fallback: UserFence, + fallback: Option>, syncs: KVec, input_count: usize, - submission: Option>, + submission: Option, result: Result, } - impl Outputs { - pub(crate) fn new( - contexts: &FenceContexts, - mut syncs: KVec, - input_count: usize, - allocation_failure: &Fence, - ) -> Result { - if input_count > syncs.len() { - return Err(EINVAL); - } - let fallback = match independent(contexts) { + pub(crate) fn new(contexts: &FenceContexts, mut syncs: KVec, + input_count: usize, allocation_failure: &Fence) -> Result + { + if input_count > syncs.len() { return Err(EINVAL); } + let fallback = match export_fallback(contexts) { Ok(fence) => fence, Err(error) => { - // Syncobjs and timeline-chain storage are already resolved. - // No work has been accepted; publish the pre-signaled ENOMEM - // fence without another allocation before returning the error. install(syncs.drain(input_count..), allocation_failure); return Err(error); } }; - Ok(Self { - fallback, - syncs, - input_count, - submission: None, - result: Err(ECANCELED), - }) - } - - pub(crate) fn inputs(&self) -> &[SyncItem] { - &self.syncs[..self.input_count] + Ok(Self { fallback: Some(fallback), syncs, input_count, + submission: None, result: Err(ECANCELED) }) } - - /// Publish the actual aggregate before enqueueing any command. - pub(crate) fn publish(&mut self, context: Arc) -> Result> { - let submission = Submission::new(self.fallback.clone(), context)?; + pub(crate) fn inputs(&self) -> &[SyncItem] { &self.syncs[..self.input_count] } + pub(crate) fn publish(&mut self, context: Arc, firmware_source: bool) -> Result { + let source = Submission::new(context, firmware_source)?; + let mut fence = self.fallback.take().ok_or(EINVAL)?; + fence.source = Some(source); + let submission = Aggregate(fence.into()); install(self.syncs.drain(self.input_count..), &submission.fence()); self.submission = Some(submission.clone()); Ok(submission) } - - /// The caller first records terminal VM errors and settles prior failed work. - pub(crate) fn finish(mut self, result: Result) -> Result { - self.result = result; - result - } + pub(crate) fn finish(mut self, result: Result) -> Result { self.result = result; result } } - impl Drop for Outputs { fn drop(&mut self) { if let Some(submission) = self.submission.take() { submission.finish_enqueue(self.result); return; } - if let Err(error) = self.result { - self.fallback.set_error(error); + if let Some(fallback) = self.fallback.take() { + if let Err(error) = self.result { fallback.set_error(error); } + install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + fallback.signal(); } - install( - self.syncs.drain(self.input_count..), - &Fence::from_fence(&self.fallback), - ); - self.fallback.signal(); } } diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index fd2cc878b534e..4ad4f8e372764 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -150,6 +150,7 @@ impl Packet { super::dependency::replay_status(fence_status(fence))?; } for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -183,6 +184,7 @@ impl Packet { for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -262,6 +264,9 @@ fn check_status(fence: &Fence) -> Result { /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { /// Device detector retained by render completions after queue teardown. + fn input_sources(&self, _inputs: &[crate::file::SyncItem]) + -> Option> { None } + fn feed(&self) -> Option> { None } @@ -430,6 +435,9 @@ impl Scheduler { // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. if !(early && self.backend.try_early(&packet)) { + if packet.completion.needs_source_ready() { + packet.completion.set_source_ready(ready.clone()); + } self.backend.count_backlog(&packet); } job.push(); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6b9e9903f3a0d..a85f68829018e 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -25,6 +25,7 @@ mod completion; pub(crate) mod context; mod data_segment; mod dependency; +mod sync_input; mod event; mod feed; mod fence; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index d2ec8da9c6082..85c65a9b22d53 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -538,6 +538,8 @@ pub(crate) struct Queue { quarantine_error: Option, failure_status_pending: bool, retirement_proved: bool, + source_readers: Arc, + source_hint: crate::g17::sync_input::ReaderHint, } impl Queue { @@ -598,6 +600,8 @@ impl Queue { quarantine_error: None, failure_status_pending: false, retirement_proved: false, + source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, + source_hint: crate::g17::sync_input::ReaderHint::new(), }) } pub(crate) fn qid(&self) -> u8 { @@ -1231,11 +1235,15 @@ impl Queue { && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() + // Borrow acquisition is serialized by the device mutex. Avoid + // touching the separate loan counter for an otherwise busy queue. + && !self.source_hint.has_readers(&self.source_readers) } pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { if !self.reusable() { return Err(EBUSY); } + self.source_hint.reset_for_owner(&self.source_readers)?; if !context.is_current() { return Err(EFAULT); } @@ -1343,7 +1351,9 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() + self.pool_phase == PoolPhase::Populated && self.pool.populated() + && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } /// A recycled killed queue still holding the backing its killed work used. pub(crate) fn recycled_pending(&self) -> bool { @@ -1780,6 +1790,17 @@ impl Queue { dependencies: &[crate::g17::fw::kick::KickDependency], host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.publish_with_native_waits(packet, dependencies, &[], host, defer) + } + + pub(in crate::g17) fn publish_with_native_waits( + &mut self, + packet: Arc, + dependencies: &[crate::g17::fw::kick::KickDependency], + native: &[crate::g17::sync_input::NativeWait], + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { use crate::g17::fw::{ channels::WorkSlot, @@ -1814,11 +1835,22 @@ impl Queue { let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; + // The first prototype only waits on an installed consumer channel. + // A retry without a published source falls back to completion on CPU. + if first && !native.is_empty() { return Err(EAGAIN); } + if native.len() > crate::g17::kick::COMPUTE_CHANNELS { return Err(EINVAL); } let entry_signal = first || dependencies .iter() .any(|dependency| dependency.qid() != qid); item_capacity(self.graph.cursors()?, entry_signal)?; + if !native.is_empty() { + let cursors = self.graph.cursors()?; + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < (if entry_signal { 4 } else { 3 }) + native.len() as u32 { + return Err(EAGAIN); + } + } let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1928,6 +1960,13 @@ impl Queue { } let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; let cursors = self.graph.cursors()?; + let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; + for (index, wait) in native.iter().enumerate() { + let record = wait.bytes(crate::g17::fw::stamp_value(kick_timestamp.get() as u32), + packet.order.sequence); + let address = self.graph.record(record_slot, 0x900 + index * 0x60, 0x60, record)?; + wait_vas.push(address, GFP_KERNEL)?; + } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { @@ -1962,15 +2001,22 @@ impl Queue { })?; } else { let target = self.pool.reserve_submission()?; - producer = self - .graph - .append(cursors.producer, descriptor_high, || { - packet.mark_published() + producer = (|| -> Result { + let mut before_descriptor = cursors.producer; + for &wait in &wait_vas { + before_descriptor = self.graph.append(before_descriptor, wait, || { + packet.mark_published(); + visible = true; + })?; + } + self.graph.append(before_descriptor, descriptor_high, || { + packet.mark_published(); + visible = true; }) - .or_else(|error| { - self.pool.cancel_unpublished(target)?; - Err(error) - })?; + })().or_else(|error| { + if !visible { self.pool.cancel_unpublished(target)?; } + Err(error) + })?; visible = true; self.unpublished_binding = false; let qos = host.qos_publish( @@ -2282,6 +2328,7 @@ impl Queue { impl Queue { pub(crate) fn can_rearm(&self) -> bool { self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } } @@ -2323,3 +2370,50 @@ impl Queue { Ok((false, retired)) } } + + +impl Queue { + /// Same immutable singleton identity and current physical epoch. + pub(crate) fn has_input_source(&self, source: &Arc, + status: &Arc, epoch: u64) -> bool + { + !self.quarantined && self.replays.len == 0 && self.active.iter().any(|active| + active.ticket.is_some_and(|ticket| ticket.epoch == epoch) + && Arc::ptr_eq(active.packet.completion.status(), status) + && active.packet.completion.is_input_source(source)) + } + + pub(crate) fn source_dependency(&mut self, source: &crate::g17::sync_input::Source, + status: &Arc, epoch: u64) -> Result> + { + for active in self.active.iter() { + if !active.packet.completion.is_input_source(&source.submission) { continue; } + if !Arc::ptr_eq(active.packet.completion.status(), status) { return Err(EINVAL); } + if self.quarantined || self.replays.len != 0 { return Err(EAGAIN); } + let ticket = active.ticket.ok_or(EAGAIN)?; + if ticket.epoch != epoch { return Err(EAGAIN); } + self.valid_ticket(active, &ticket)?; + if active.packet.cancel_requested() { return Err(ECANCELED); } + source.submission.source_status()?; + let dependency = crate::g17::fw::kick::KickDependency::new(self.qid(), + crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?).ok_or(EIO)?; + let hold = self.source_hint.hold(&self.source_readers)?; + return Ok(Some((active.published, dependency, + self.graph.support.gpu_va() + STAMP as u64, + u32::from(self.qid()), hold))); + } + Ok(None) + } + + /// Preparing a possible wait does not create a firmware consumer. Mark it + /// only once the consumer's first item is visible, under the device mutex. + pub(crate) fn note_input_consumer(&self, source: &crate::g17::sync_input::Source) { + for active in self.active.iter() { + if active.packet.completion.is_input_source(&source.submission) { + active.packet.completion.note_firmware_consumer(); + break; + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 96e566d3971ff..777c4bc1440ac 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -10,7 +10,7 @@ use super::{ command::{Parser, UscWindow, Validated}, completion::{Completion, Destinations, PreparedPublication}, context::Context, - fence::{Outputs, Submission}, + fence::Outputs, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, validation::Proof, }; @@ -179,7 +179,7 @@ impl Queue { command: Validated, order: Order, guard: mmu::VmJobGuard, - aggregate: &Arc, + aggregate: &super::fence::Aggregate, inputs: &[file::SyncItem], destinations: Destinations, ) -> Result { @@ -198,6 +198,22 @@ impl Queue { let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; self.ensure_compute(scratch.pages())?; } + // Only convert inputs when no independent host-prefix wait could keep + // the retained source bookkeeping queued behind an external signal. + let sources = if engine == Engine::Compute && !inputs.is_empty() + && !destinations.has_outputs() + && self.frontiers[0].iter().all(|(sequence, fences)| + !Order::contains(order.wait_through[0], *sequence) + || job::fence_status(&fences.completed) > 0) + && self.frontiers[1].iter().all(|(sequence, fences)| + !Order::contains(order.wait_through[1], *sequence) + || fences.ready.raw() != fences.completed.raw() + || job::fence_status(&fences.completed) > 0) + && self.failures.iter().enumerate().all(|(index, failure)| + failure.as_ref().is_none_or(|(sequence, _)| + !Order::contains(order.wait_through[index], *sequence))) + { self.backend.input_sources(inputs) } else { None }; + // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. let host_predecessors = self.frontiers[index] @@ -217,6 +233,7 @@ impl Queue { engine == Engine::Compute, host_predecessors.clone().next().is_some(), self.backend.feed(), + sources.clone(), )?; let publication = completion.publication() .map(|target| PreparedPublication::new(target, host_predecessors)) @@ -294,10 +311,19 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } + if sources.as_ref().is_some_and(|sources| sources.contains_input(fence)) { + continue; + } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } + if let Some(sources) = &sources { + for ready in sources.publication_waits() { + dependencies.push(ready.clone(), GFP_KERNEL)?; + checked_inputs.push(ready.clone(), GFP_KERNEL)?; + } + } let early = dependencies.is_empty() && self.frontiers[index].len() < EARLY_DEPTH; let packet = Packet::new( @@ -416,7 +442,10 @@ impl crate::queue::Queue for Queue { for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } } let mut guards = guards.into_iter(); - let aggregate = outputs.publish(self.context.clone())?; + let firmware_source = pending.len() == 1 + && matches!(&pending[0].0, Validated::Compute { .. }) + && !pending[0].2.has_outputs(); + let aggregate = outputs.publish(self.context.clone(), firmware_source)?; for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index bd95148e9eaf0..d0f3f3cfb2eb2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -959,6 +959,7 @@ impl Backend { /// The physical namespace bounds the batch independently of open logical queues. /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { + has_source_completions: bool, values: KVVec, /// Exited owners detached from idle retained compute queues and the backing /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. @@ -994,6 +995,7 @@ impl DeferredBatch { 0 }; Ok(Self { + has_source_completions: false, values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, @@ -1051,6 +1053,12 @@ impl DeferredBatch { completion.status().record(*error); } } + match &value { + Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) + | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => + { completion.note_input_error(*error); }, + _ => {}, + } self.values.push_within_capacity(value).map_err(|_| EIO)?; // A later publisher can borrow this pair before finish signals its old // fence. Publish the exact completion's witness only after collection @@ -1062,11 +1070,29 @@ impl DeferredBatch { ) = self.values.last() { completion.note_retirement_queued(); + self.has_source_completions |= completion.sources().is_some(); } Ok(()) } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { + use super::completion::Deferred; + if self.has_source_completions { + for _ in 0..self.values.len() { + let mut changed = false; + for value in &self.values { + if let Deferred::Retired(completion, Ok(_)) + | Deferred::Drained(completion, Ok(_)) = value + { + if let Err(error) = completion.check_source_inputs() { + changed |= completion.note_input_error(error); + } + } + } + if !changed { break; } + } + } + self.has_source_completions = false; for value in self.values.drain_all() { value.finish(); } @@ -1140,11 +1166,75 @@ impl compute::Host for ComputeHost<'_> { } impl super::Firmware { - fn publish_compute( + #[inline] + fn publish_compute(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if packet.completion.sources().is_some() { + self.publish_compute_inputs(owner, packet, dependencies, deferred) + } else { + self.publish_compute_raw(owner, packet, dependencies, &[], deferred) + } + } + + #[inline(never)] + fn publish_compute_inputs(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if self.recovery.pending() || super::recovery::Memory::recovery_state(&self.init)? != 0 { + return Err(EAGAIN); + } + let sources = packet.completion.sources().ok_or(EINVAL)?.clone(); + sources.check()?; + let mut merged = super::dependency::Frontier::new(); + for dependency in dependencies { merged.insert(0, *dependency)?; } + let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + let mut native: KVec = + KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + for source in &sources.sources { + if source.submission.source_status()? { continue; } + let found = self.queues.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find_map(|queue| match queue.source_dependency(source, packet.completion.status(), + self.recovery.generation()) { + Ok(None) => None, + result => Some(result), + }).ok_or(EAGAIN)??; + let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { + return Err(EAGAIN); + }; + merged.insert(sequence, dependency)?; + let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; + if let Some(old) = native.iter_mut().find(|old| + old.dependency.qid() == dependency.qid()) { + if sequence > old.sequence { *old = wait; } + } else { + native.push(wait, GFP_KERNEL)?; + } + holds.push(hold, GFP_KERNEL)?; + } + let result = self.publish_compute_raw(owner, packet.clone(), merged.as_slice(), + native.as_slice(), deferred); + if packet.is_published() { + for queue in self.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) + { + for source in &sources.sources { queue.note_input_consumer(source); } + } + sources.set_holds(holds); + super::sync_input::note_published(); + super::sync_input::note_native(native.len()); + } + result + } + + fn publish_compute_raw( &mut self, owner: u64, packet: Arc, dependencies: &[super::fw::kick::KickDependency], + native: &[super::sync_input::NativeWait], deferred: &mut DeferredBatch, ) -> Result { if self.recovery.pending() { @@ -1183,9 +1273,12 @@ impl super::Firmware { accounting: &mut queues.accounting, generation: self.recovery.generation(), }; - queue.publish(packet, dependencies, &mut host, &mut |event| { - deferred.push(event) - }) + if native.is_empty() { + queue.publish(packet, dependencies, &mut host, &mut |event| deferred.push(event)) + } else { + queue.publish_with_native_waits(packet, dependencies, native, + &mut host, &mut |event| deferred.push(event)) + } } } @@ -1442,6 +1535,27 @@ impl Backend { } } + /// A scheduled dependency may refer to a previous recovery epoch. Do not + /// acquire consumer capacity until every live source is published now. + fn sources_published(&self, packet: &super::job::Packet) -> Result { + let Some(sources) = packet.completion.sources() else { return Ok(true); }; + sources.check()?; + let state = self.shared.state.lock(); + let firmware = state.as_deref().ok_or(ENODEV)?; + if firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { return Ok(false); } + for source in &sources.sources { + if source.submission.source_status()? { continue; } + if !firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&source.submission, + packet.completion.status(), firmware.recovery.generation())) + { return Ok(false); } + } + Ok(true) + } + /// Called after taking this packet's publication claim. All waits and work /// state allocation precede the device mutex; only the checked epoch commits. fn compute_packet( @@ -1468,6 +1582,11 @@ impl Backend { let admission = Admission::new(deadline, &check); loop { admission.check()?; + if !self.sources_published(packet)? { + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } // All DRM dependencies were satisfied before this scheduler callback. // Recheck on every retry: pressure may yield an idle binding between // admission and publication without changing this logical queue. diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index c75b525413ee8..b393afd9c8df2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -112,6 +112,55 @@ impl Backend { } impl job::Backend for Backend { + fn input_sources(&self, inputs: &[crate::file::SyncItem]) + -> Option> + { + use crate::g17::{fence, sync_input}; + let mut heads = KVec::new(); + let mut sources: KVec = KVec::new(); + let mut publication_waits = KVec::new(); + for input in inputs.iter().filter_map(|input| input.fence.as_ref()) { + let status = job::fence_status(input); + if status > 0 { continue; } + if status < 0 { return None; } + let leaves = sync_input::collect(input)?; + for leaf in &leaves { + if leaf.status > 0 { continue; } + if leaf.status < 0 { return None; } + // SAFETY: Collection owns each leaf until this lookup finishes. + let submission = unsafe { fence::exported_source_raw(leaf.fence) }?; + if submission.source_status().ok()? { continue; } + if !submission.same_vm(self.context.status()) { return None; } + if sources.iter().any(|known| Arc::ptr_eq(&known.submission, &submission)) { + continue; + } + if sources.len() == crate::g17::fw::kick::KICK_BARRIERS_MAX { return None; } + let published = { + let state = self.shared.state.lock(); + let firmware = state.as_deref()?; + if self.shared.crashed.load(Ordering::Acquire) || firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init).ok()? != 0 + { return None; } + firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&submission, self.context.status(), + firmware.recovery.generation())) + }; + if !published { + if submission.source_status().ok()? { continue; } + // The existing scheduler fence becomes ready after actual + // publication, not GPU completion. No binding loan or + // consumer hardware capacity is acquired while waiting. + publication_waits.push(submission.ready()?, GFP_KERNEL).ok()?; + } + sources.push(sync_input::Source { submission }, GFP_KERNEL).ok()?; + } + heads.push(input.clone(), GFP_KERNEL).ok()?; + } + if sources.is_empty() { return None; } + sync_input::Sources::new(heads, sources, publication_waits).ok() + } + fn feed(&self) -> Option> { Some(self.shared.feed.clone()) } diff --git a/drivers/gpu/drm/asahi/g17/sync_input.c b/drivers/gpu/drm/asahi/g17/sync_input.c new file mode 100644 index 0000000000000..e6b91988c3c28 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include "sync_input.h" + +/* Called without the device mutex: status polling may run fence callbacks. */ +int asahi_g17_collect_fences(struct dma_fence *head, + struct asahi_g17_fence_leaf *out, size_t capacity) +{ + struct dma_fence_unwrap cursor; + struct dma_fence *leaf; + size_t count = 0; + int error = 0; + + dma_fence_unwrap_for_each(leaf, &cursor, head) { + if (dma_fence_is_container(leaf)) { + error = -EOPNOTSUPP; + break; + } + if (count == capacity) { + error = -ENOSPC; + break; + } + out[count].fence = dma_fence_get(leaf); + out[count].status = dma_fence_get_status(leaf); + ++count; + } + /* A bounded early exit still owns the iterator's current chain. */ + dma_fence_put(cursor.chain); + if (error) { + while (count) + dma_fence_put(out[--count].fence); + return error; + } + return count; +} + +void asahi_g17_put_fence(struct dma_fence *fence) +{ + dma_fence_put(fence); +} diff --git a/drivers/gpu/drm/asahi/g17/sync_input.h b/drivers/gpu/drm/asahi/g17/sync_input.h new file mode 100644 index 0000000000000..2b6e80f4b7f66 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_INPUT_H +#define ASAHI_G17_SYNC_INPUT_H + +#include + +struct dma_fence; + +struct asahi_g17_fence_leaf { + struct dma_fence *fence; + int status; +}; + +int asahi_g17_collect_fences(struct dma_fence *head, + struct asahi_g17_fence_leaf *out, size_t capacity); +void asahi_g17_put_fence(struct dma_fence *fence); + +#endif diff --git a/drivers/gpu/drm/asahi/g17/sync_input.rs b/drivers/gpu/drm/asahi/g17/sync_input.rs new file mode 100644 index 0000000000000..1cd9a9ba761aa --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded input-fence components. Collection happens before the device mutex, +//! because polling a container's status can signal it and run callbacks. + +use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; + +#[repr(C)] +pub(super) struct Leaf { + pub(super) fence: *mut bindings::dma_fence, + pub(super) status: i32, +} + +unsafe extern "C" { + fn asahi_g17_collect_fences(head: *mut bindings::dma_fence, + out: *mut Leaf, capacity: usize) -> i32; + fn asahi_g17_put_fence(fence: *mut bindings::dma_fence); +} + +impl Drop for Leaf { + fn drop(&mut self) { + // SAFETY: Successful collection transfers exactly one owned reference. + unsafe { asahi_g17_put_fence(self.fence) }; + } +} + +/// Allocation, unsupported containers, or a large prefix retain normal waits. +pub(super) fn collect(input: &Fence) -> Option> { + let mut capacity = 4; + loop { + let mut leaves = KVec::with_capacity(capacity, GFP_KERNEL).ok()?; + // SAFETY: The input owns its fence. The C helper writes at most capacity + // fully initialized records and releases every reference on failure. + let count = unsafe { + asahi_g17_collect_fences(input.raw(), leaves.as_mut_ptr(), capacity) + }; + if count == ENOSPC.to_errno() && capacity < 128 { + capacity = (capacity * 4).min(128); + continue; + } + if count < 0 { return None; } + // SAFETY: The helper returned this many initialized, owned records. + unsafe { leaves.set_len(count as usize) }; + return Some(leaves); + } +} + + +use super::fence::Submission; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::{new_mutex, sync::{Arc, Mutex}}; + +pub(super) struct Source { + pub(super) submission: Arc, +} + +/// The counter is independent of a physical queue's allocation and survives +/// processor stop until the last consumer releases its binding loan. +pub(super) struct Hold(Arc); +impl Hold { + fn new(readers: &Arc) -> Result { + readers.fetch_update(Ordering::AcqRel, Ordering::Acquire, + |count| count.checked_add(1)).map_err(|_| EOVERFLOW)?; + Ok(Self(readers.clone())) + } +} +impl Drop for Hold { + fn drop(&mut self) { self.0.fetch_sub(1, Ordering::AcqRel); } +} + +/// A new owner starts with no loans. Acquiring a loan requires exclusive access +/// and arms the hint; eligibility queries stay read-only. A validated owner +/// transition can clear the hint after all independent loans have been released. +#[repr(transparent)] +pub(super) struct ReaderHint { + possible: bool, +} +impl ReaderHint { + pub(super) const fn new() -> Self { + Self { possible: false } + } + pub(super) fn hold(&mut self, count: &Arc) -> Result { + let hold = Hold::new(count)?; + self.possible = true; + Ok(hold) + } + pub(super) fn has_readers(&self, count: &AtomicU32) -> bool { + self.possible && count.load(Ordering::Acquire) != 0 + } + pub(super) fn reset_for_owner(&mut self, count: &AtomicU32) -> Result { + if self.has_readers(count) { return Err(EBUSY); } + self.possible = false; + Ok(()) + } +} + +#[pin_data] +pub(super) struct Sources { + inputs: KVec, + publication_waits: KVec, + pub(super) sources: KVec, + #[pin] + holds: Mutex>, +} +impl Sources { + pub(super) fn new(inputs: KVec, sources: KVec, + publication_waits: KVec) -> Result> { + CONVERTED.fetch_add(1, Ordering::Relaxed); + if !publication_waits.is_empty() { PENDING.fetch_add(1, Ordering::Relaxed); } + Arc::pin_init(pin_init!(Self { + inputs, sources, publication_waits, + holds <- new_mutex!(KVec::new(), "G17 source binding loans"), + }), GFP_KERNEL) + } + pub(super) fn contains_input(&self, input: &Fence) -> bool { + self.inputs.iter().any(|fence| fence.raw() == input.raw()) + } + pub(super) fn publication_waits(&self) -> &[Fence] { &self.publication_waits } + pub(super) fn check(&self) -> Result { + for source in &self.sources { source.submission.source_status()?; } + Ok(()) + } + pub(super) fn set_holds(&self, holds: KVec) { + let old = core::mem::replace(&mut *self.holds.lock(), holds); + drop(old); + } + pub(super) fn release(&self) { self.set_holds(KVec::new()); } +} + + +static CONVERTED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static PUBLISHED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static PENDING: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(super) fn pending_count() -> u64 { PENDING.load(Ordering::Relaxed) } +pub(super) fn note_published() { PUBLISHED.fetch_add(1, Ordering::Relaxed); } +pub(super) fn counters() -> (u64, u64) { + (CONVERTED.load(Ordering::Relaxed), PUBLISHED.load(Ordering::Relaxed)) +} + +/// A revalidated compute completion dependency for a direct work descriptor. +#[derive(Clone, Copy)] +pub(super) struct NativeWait { + pub(super) sequence: u64, + pub(super) dependency: super::fw::kick::KickDependency, + pub(super) stamp_va: u64, + pub(super) event_slot: u32, +} +impl NativeWait { + pub(super) fn bytes(&self, consumer_stamp: u32, uuid: u64) -> [u8; 0x48] { + let mut out = [0; 0x48]; + let timestamp = self.dependency.word() & ((1u64 << 40) - 1); + for (offset, value) in [ + (0, 4u32), + (0x14, u32::from(self.dependency.qid())), + (0x18, super::fw::stamp_value(timestamp as u32)), + (0x24, self.event_slot), + (0x28, consumer_stamp), + (0x34, 1), + ] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(4, self.stamp_va), (0xc, timestamp), (0x2c, uuid)] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + out + } +} + +static NATIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(super) fn note_native(count: usize) { NATIVE.fetch_add(count as u64, Ordering::Relaxed); } +pub(super) fn native_count() -> u64 { NATIVE.load(Ordering::Relaxed) } + +#[kunit_tests(asahi_g17_input_wait)] +mod tests { + use super::*; + use crate::g17::fw::kick::{KickDependency, KickTimestamp}; + + #[test] + fn completion_record_preserves_distinct_event_and_queue_domains() { + let wait = NativeWait { + sequence: 1, + dependency: KickDependency::new(37, + KickTimestamp::new(0x12_3456_789a).unwrap()).unwrap(), + stamp_va: 0x1122_3344_5566_7788, + event_slot: 7, + }; + // Protocol fixture includes the unaligned stamp pointer, full 40-bit + // timestamp, wrapped 24-bit stamp and independent legacy event slot. + let expected: [u8; 0x48] = [ + 4, 0, 0, 0, 0x88, 0x77, 0x66, 0x55, + 0x44, 0x33, 0x22, 0x11, 0x9a, 0x78, 0x56, 0x34, + 0x12, 0, 0, 0, 37, 0, 0, 0, + 0, 0x9a, 0x78, 0x56, 0, 0, 0, 0, + 0, 0, 0, 0, 7, 0, 0, 0, + 0, 0xcc, 0xbb, 0xaa, 8, 7, 6, 5, + 4, 3, 2, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + ]; + assert_eq!(wait.bytes(0xaabb_cc00, 0x0102_0304_0506_0708), expected); + } + + #[test] + fn failed_loan_acquisition_never_wraps_or_releases_another_reader() { + let readers = Arc::new(AtomicU32::new(u32::MAX - 1), GFP_KERNEL).unwrap(); + let last = Hold::new(&readers).unwrap(); + assert!(matches!(Hold::new(&readers), Err(error) if error == EOVERFLOW)); + assert_eq!(readers.load(Ordering::Acquire), u32::MAX); + drop(last); + assert_eq!(readers.load(Ordering::Acquire), u32::MAX - 1); + } + + #[test] + fn loan_outlives_original_queue_counter_reference() { + let readers = Arc::new(AtomicU32::new(0), GFP_KERNEL).unwrap(); + let mut hint = ReaderHint::new(); + let first = hint.hold(&readers).unwrap(); + let second = hint.hold(&readers).unwrap(); + drop(readers); + assert_eq!(second.0.load(Ordering::Acquire), 2); + drop(first); + assert_eq!(second.0.load(Ordering::Acquire), 1); + drop(second); + } + + #[test] + fn reader_guard_tracks_partial_release_and_rearms_after_zero() { + let readers = Arc::new(AtomicU32::new(0), GFP_KERNEL).unwrap(); + let mut hint = ReaderHint::new(); + assert!(!hint.has_readers(&readers)); + let first = hint.hold(&readers).unwrap(); + let second = hint.hold(&readers).unwrap(); + assert!(hint.has_readers(&readers)); + assert!(matches!(hint.reset_for_owner(&readers), Err(error) if error == EBUSY)); + drop(first); + assert!(hint.has_readers(&readers)); + drop(second); + assert!(!hint.has_readers(&readers)); + hint.reset_for_owner(&readers).unwrap(); + let next = hint.hold(&readers).unwrap(); + assert!(hint.has_readers(&readers)); + drop(next); + assert!(!hint.has_readers(&readers)); + } +} From 89c8873651b729fdd4e981e87b9d0df0d0ad0017 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:29:21 -0400 Subject: [PATCH 09/64] drm/asahi: treat a firmware source as complete once its retirement is queued Compute retirement removes the producer from its queue's active list under the device mutex, but signals the producer's fence only after the lock is dropped. A consumer published in between found neither an active producer nor a completed one, fell back to the scheduler, and every later command in the chain then waited for its producer's publication through the scheduler. Mark the singleton firmware source retired as its successful retirement is queued (errors are already recorded there), after rechecking its own inputs. sync.signal queue-to-queue (timeline, 2 queues): scheduler fallbacks for chain consumers 57% -> 0.1%. (cherry picked from commit a3be7f63500c76d117000ee1474713e27889aabe) --- drivers/gpu/drm/asahi/g17/completion.rs | 7 +++++++ drivers/gpu/drm/asahi/g17/fence.rs | 12 +++++++++++- drivers/gpu/drm/asahi/g17/runtime.rs | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 406efc0e221fe..5936bcb1b933a 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -377,6 +377,13 @@ impl Completion { pub(super) fn note_input_error(&self, error: Error) -> bool { self.output.output().member.note_error(error) } + /// Called under the device mutex as a successful retirement is queued. + pub(super) fn note_source_retired(&self) { + match self.check_source_inputs() { + Ok(()) => self.output.output().member.note_retired(), + Err(error) => { self.note_input_error(error); } + } + } pub(super) fn check_source_inputs(&self) -> Result { if let Some(sources) = &self.sources { sources.check()?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 25e5f6b04d558..da883cede52df 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -121,6 +121,10 @@ pub(crate) struct Submission { /// Live Member objects, plus the enqueue guard, independently of signalling. owners: AtomicU8, firmware_source: bool, + /// Set under the device mutex when the only member's successful retirement + /// is queued, before its fence is signalled off-lock: a consumer published + /// in between needs no wait on it. + retired: AtomicBool, } impl Submission { @@ -135,10 +139,12 @@ impl Submission { error: AtomicI32::new(0), owners: AtomicU8::new(1), firmware_source, + retired: AtomicBool::new(false), }), GFP_KERNEL) } pub(super) fn source_status(&self) -> Result { - let complete = self.remaining.load(Ordering::Acquire) == 0; + let complete = self.remaining.load(Ordering::Acquire) == 0 + || self.retired.load(Ordering::Acquire); let error = self.error.load(Ordering::Acquire); if error != 0 { return Err(super::dependency::inherited_error(Error::from_errno(error))); } Ok(complete) @@ -212,6 +218,10 @@ impl Member { } pub(super) fn set_source_ready(&self, ready: Fence) { self.aggregate.source().set_ready(ready); } pub(super) fn note_error(&self, error: Error) -> bool { self.aggregate.source().note_error(error) } + pub(super) fn note_retired(&self) { + let source = self.aggregate.source(); + if source.firmware_source { source.retired.store(true, Ordering::Release); } + } pub(crate) fn complete(&self, result: Result) { if !self.complete.swap(true, Ordering::AcqRel) { self.aggregate.source().member_done(&self.aggregate.0, result); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d0f3f3cfb2eb2..736fed78b0cc1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1057,6 +1057,8 @@ impl DeferredBatch { Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => { completion.note_input_error(*error); }, + Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => + completion.note_source_retired(), _ => {}, } self.values.push_within_capacity(value).map_err(|_| EIO)?; From 3ff4f4baa05a88d5f99d2fee76089cf72b5617a4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:29:21 -0400 Subject: [PATCH 10/64] drm/asahi: publish G17 commands from the ioctl at any queue depth Early publication was limited to queues with fewer than two outstanding commands, so every deeper queue paid a scheduler worker hop per command and cross-queue chains advanced one scheduler round trip per link. The backend already falls back to the scheduler behind any backlog and on every capacity miss, which preserves submission order, so the depth limit only added latency. sync.signal queue-to-queue (timeline, 2 queues) 94.1 -> 56.2 us/hop together with the preceding change; compute and render depth rows unchanged. (cherry picked from commit 8508510641edcedb5ae1417f560cdff4da646bda) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 777c4bc1440ac..c959f8bc8c4b8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -27,7 +27,6 @@ use kernel::{ xarray, }; -const EARLY_DEPTH: usize = 2; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -324,8 +323,10 @@ impl Queue { checked_inputs.push(ready.clone(), GFP_KERNEL)?; } } - let early = dependencies.is_empty() - && self.frontiers[index].len() < EARLY_DEPTH; + // Publish from the ioctl whenever nothing has to wait, at any depth. + // The backend falls back to the scheduler behind any backlog or on any + // capacity miss, so submission order is preserved. + let early = dependencies.is_empty(); let packet = Packet::new( id, self.context.clone(), From a0dfc3c0b5a4fced66119fded07ed141990a9025 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:49:26 -0400 Subject: [PATCH 11/64] drm/asahi: map G17 compute descriptors cached for the firmware Render descriptors already come from the retained object pool with a firmware-cached mapping; compute descriptors were mapped uncached, so the firmware read every compute descriptor from memory on each command. Allocate the compute descriptor ring from the same pool (fully cached prefix, retained until processor stop) and give the hardware an uncached canonical low alias for the descriptor address it fetches. compute.light d64 27.4 -> 24.3 us/submit, queue-to-queue hop 58.1 -> 54.8 us; compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit c85c050ae2ef77cbeebddb70c6602db477675090) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 46 +++++++++++++--------- drivers/gpu/drm/asahi/g17/runtime.rs | 9 +++-- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 85c65a9b22d53..3547f312224b4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -15,7 +15,7 @@ use crate::g17::{ queue::*, variant::PointerLayout, }, - object::{Allocator, CpuMap, KernelObject}, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; use crate::{hw::t8140::dynamic, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -83,24 +83,32 @@ pub(crate) struct Binding { /// Descriptor ring of one compute QID with its kernel-VM alias. Built without /// the device mutex; installed into a vacant graph before an owner binds. pub(crate) struct DescriptorRing { - object: KernelObject, - high: mmu::KernelMapping, + // Drop the GPU alias before the retained storage returns to its pool. + low: mmu::KernelMapping, + object: PooledObject, } impl DescriptorRing { - pub(crate) fn new(alloc: &Allocator<'_>) -> Result { - let page = mmu::UAT_PGSZ as u64; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let object = alloc.lower( - DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - page, - prot, + /// Firmware reads every descriptor through a cached kernel mapping, as it + /// does for render. The pool keeps that mapping until processor stop; the + /// hardware reads the same pages through an uncached canonical low alias. + pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { + let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; + let object = objects.allocate( + alloc, + Family::Descriptor, + size, + size, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; - Ok(Self { object, high }) + let low = object.alias_in( + alloc.uat.kernel_lower_vm(), + dynamic::LOWER, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + )?; + Ok(Self { low, object }) } } @@ -127,6 +135,7 @@ struct Graph { impl Graph { fn new( alloc: &Allocator<'_>, + objects: &Arc, context: &Context, pointers: PointerLayout, operand_state: &'static OperandState, @@ -134,7 +143,7 @@ impl Graph { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let ring = DescriptorRing::new(alloc)?; + let ring = DescriptorRing::new(alloc, objects)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(operand_state.size, page, prot, cpu)?; @@ -250,12 +259,12 @@ impl Graph { fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; - Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) + Ok((ring.object.gpu_va() + offset, ring.low.iova() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { let ring = self.ring.as_ref().ok_or(EAGAIN)?; ring.object - .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) + .map_alias(vm, ring.low.iova(), mmu::PROT_GPU_SHARED_RO) } /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed @@ -547,6 +556,7 @@ impl Queue { /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( alloc: &Allocator<'_>, + objects: &Arc, backing: Backing, owner: u64, context: Arc, @@ -554,7 +564,7 @@ impl Queue { operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 736fed78b0cc1..d94e344c6f713 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -790,7 +790,7 @@ impl Backend { admission.check()?; let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; - let (uat, reservation) = { + let (uat, objects, reservation) = { let mut state = self.shared.state.lock(); admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -814,7 +814,7 @@ impl Backend { match rebound { Rebind::Done => return Ok(()), Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } - Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), + Rebind::Rearm(qid) => (firmware.uat.clone(), firmware.pool.clone(), Err(qid)), Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, @@ -838,7 +838,7 @@ impl Backend { } Err(error) => return Err(error), }; - (firmware.uat.clone(), Ok(reservation)) + (firmware.uat.clone(), firmware.pool.clone(), Ok(reservation)) } } }; @@ -853,7 +853,7 @@ impl Backend { // queue may have been taken or re-armed by another owner since. let built = (|| -> Result<_> { let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc)?)) + Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) })(); let (pages, ring) = match built { Err(error) if error == ENOSPC => { @@ -908,6 +908,7 @@ impl Backend { self.compute_scheduler(&alloc, &lease, admission)?; let queue = compute::Queue::new( &alloc, + &objects, backing, self.owner, self.context.clone(), From a36e352d8564947071fef7e11dba709f399cd106 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:38:10 -0400 Subject: [PATCH 12/64] drm/asahi: size the G17 compute QID budget by open logical queues The compute channel budget was static: compute could only take a QID while every one of the 58 possible execution contexts could still install a render pair, which always left exactly ten compute channels. A client without a channel then made every idle channel of every live client shareable before binding one, so sixteen compute processes evicted each other on nearly every submission (map/unmap and TLB invalidation churn, ComputeReserve refused 52k times in one benchmark pass). Installed QIDs never change engine before processor stop, so a compute channel still permanently narrows render. Size the render reserve by the logical queues that are actually open plus 24 queues of headroom, and refuse a new logical queue only when its dedicated render pair would no longer fit. With every context open this is the previous rule. Rebinding now prefers a channel of a closed owner, then a vacant one to re-arm, then a fresh QID, and only then shares the least recently used idle channel of a single live client. gpubench on Neo (us/submit, matched baseline): compute.light procs d16 92.0 -> 33.4; compute.light queue rows unchanged (d64 27.4 -> 26.5). Stress gate passes. (cherry picked from commit bf7ae5bf04b7d3f14b5ec94ef1f8c1434b4fe494) --- drivers/gpu/drm/asahi/g17/kick.rs | 121 ++++++++++++++------- drivers/gpu/drm/asahi/g17/queue/compute.rs | 24 +++- drivers/gpu/drm/asahi/g17/runtime.rs | 95 +++++++++++++--- 3 files changed, 178 insertions(+), 62 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index c337a8b298f19..87a90516d1b42 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -24,22 +24,38 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// Minimum render-channel capacity protected from compute-only allocation. -// Protect one tiling/fragment pair for each logical execution context, plus -// the two unpublished bootstrap reservations. The remaining ten compute -// channels are shared after exact retirement, rather than pinned per client. -const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; -pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; -/// Logical queues that can each hold a render pair: what the render reserve leaves beside the -/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone -/// (every execution context with no floor). +/// Render identities that must stay reachable for `queues` logical queues: +/// one tiling/fragment pair each, plus the two bootstrap reservations. +/// +/// Installed identities never change engine before processor stop, so a +/// compute channel permanently narrows what render can still install. Compute +/// may take a free QID only while every open logical queue, and +/// [`RENDER_HEADROOM`] queues opened later, can still install a dedicated +/// pair; logical-queue admission refuses a queue whose pair no longer fits. +pub(super) const fn render_reserve(queues: usize) -> usize { + let queues = queues + RENDER_HEADROOM; + let max = crate::mmu::MAX_EXECUTION_CONTEXTS as usize; + 2 * if queues < max { queues } else { max } + 2 +} +/// Logical queues that may still open after compute growth without losing a +/// dedicated render pair. +const RENDER_HEADROOM: usize = 24; +/// The render reserve with every execution context open. +const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize); +/// Compute channels that stay reachable with every execution context open. +#[cfg(CONFIG_KUNIT)] +pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE; +/// Upper bound on distinct physical compute channels named by one kick. +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT; +/// Maximum logical queues after firmware-owned QIDs are excluded. pub(super) const fn queue_limit(floor: usize) -> u32 { - ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 + ((FULL_RENDER_RESERVE - 2 - floor) / 2) as u32 } const _: () = { assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); - assert!(COMPUTE_CHANNELS >= 4); + // Compute keeps this many channels even with every execution context open. + assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); }; @@ -85,7 +101,7 @@ impl QueueIds { publication: 0, leased: 0, // The reserve must keep render capacity above the floor. - floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, + floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } @@ -113,6 +129,53 @@ impl QueueIds { owner: u64, engine: DataMaster, preferred: Option, + ) -> Result { + self.reserve_for(owner, engine, preferred, crate::mmu::MAX_EXECUTION_CONTEXTS as usize) + } + + fn render_ids(&self) -> usize { + self.entries + .iter() + .flatten() + .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) + .count() + } + + /// Whether `queues` open logical queues can each still install a dedicated + /// render pair, beside the bootstrap reservations and installed compute. + pub(crate) fn render_fits(&self, queues: usize) -> bool { + self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2 + } + + /// Whether a fresh compute identity fits beside the render reserve of + /// `queues` open logical queues, without reserving it. + pub(crate) fn compute_fits(&self, queues: usize) -> bool { + let initial = self.leased == QID_COUNT - self.floor + && self.initial_render.iter().flatten().any(|id| self.can_cancel_unpublished(*id)); + self.compute_room(queues, usize::from(initial)) + } + + fn compute_room(&self, queues: usize, replacement: usize) -> bool { + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let render = self.render_ids(); + QID_COUNT - self.floor - self.leased + replacement + > render_reserve(queues) + .min(FULL_RENDER_RESERVE - self.floor) + .saturating_sub(render - replacement) + } + + /// As [`Self::reserve`], with the compute reserve sized for `queues` open + /// logical queues instead of every possible execution context. + pub(crate) fn reserve_for( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -129,10 +192,8 @@ impl QueueIds { Err(EINVAL) }; } - // The QIDs below the floor are never the host's; the reserve shrinks - // with them, so the compute channel count stays the same. + // The QIDs below the floor are never available to the host. let total = QID_COUNT - self.floor; - let reserve = COMPUTE_RENDER_RESERVE - self.floor; let initial = if engine == DataMaster::Compute && self.leased == total { self.initial_render .iter() @@ -145,27 +206,9 @@ impl QueueIds { } else { None }; - // Existing render identities already satisfy this reservation. Counting - // them twice can refuse compute even while physical QIDs remain free. // Published identities always retain their original lifetime rules. - if engine == DataMaster::Compute { - let render = self - .entries - .iter() - .flatten() - .filter(|entry| { - matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) - }) - .count(); - // Replacing an initial reservation consumes one render identity. - // Include that transition when protecting render's minimum, and - // count the replacement as available compute capacity. - let replacement = usize::from(initial.is_some()); - if total - self.leased + replacement - <= reserve.saturating_sub(render - replacement) - { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); - } + if engine == DataMaster::Compute && !self.compute_room(queues, usize::from(initial.is_some())) { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); } let index = preferred .map(|index| usize::from(index) + self.floor) @@ -249,10 +292,10 @@ mod accounting_tests { #[test] fn render_identities_satisfy_the_compute_reserve() { let mut ids = table(); - for owner in 1..=COMPUTE_RENDER_RESERVE as u64 { + for owner in 1..=FULL_RENDER_RESERVE as u64 { ids.reserve(owner, DataMaster::Tiling, None).unwrap(); } - for owner in COMPUTE_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + for owner in FULL_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.leased, QID_COUNT); @@ -263,14 +306,14 @@ mod accounting_tests { #[test] fn compute_cannot_consume_the_render_minimum() { let mut ids = table(); - for owner in 1..=(QID_COUNT - COMPUTE_RENDER_RESERVE) as u64 { + for owner in 1..=(QID_COUNT - FULL_RENDER_RESERVE) as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); ids.cancel_unpublished(tiling).unwrap(); - assert_eq!(ids.leased, QID_COUNT - COMPUTE_RENDER_RESERVE); + assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); } #[test] diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 3547f312224b4..69841e5cd2aeb 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -534,6 +534,7 @@ pub(crate) struct Queue { published_config: Option, ordinal: u32, submitted: u32, + last_published: u64, last_end: u64, released: bool, retirement_ready: bool, @@ -597,6 +598,7 @@ impl Queue { published_config: None, ordinal: 0, submitted: 0, + last_published: 0, last_end: 0, released: false, retirement_ready: false, @@ -1926,6 +1928,7 @@ impl Queue { priority: policy.priority(), }; let published = host.next_compute_publication()?; + self.last_published = published; self.active .push(Active { packet: packet.clone(), @@ -2323,16 +2326,27 @@ impl Queue { /// Logical queue/context ownership is unchanged. The existing binding is /// retained through the successor's exact replacement witness. pub(crate) fn share_idle(&mut self) -> Result { - if !self.installed || self.owner.is_none() || !self.retirement_ready - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.previous.is_some() || self.pool_phase != PoolPhase::Populated - { + if !self.shareable() { return Ok(false); } self.release_owner()?; Ok(self.reusable()) } + + /// An open client's idle channel that [`Self::share_idle`] can hand over. + pub(crate) fn shareable(&self) -> bool { + self.installed && self.owner.is_some() && self.retirement_ready + && !self.growing && !self.quarantined && !self.awaiting_witness + && self.exit.is_none() && self.active.len == 0 && self.replays.len == 0 + && self.previous.is_none() && self.pool_phase == PoolPhase::Populated + && !self.source_hint.has_readers(&self.source_readers) + } + + /// Device publication order of this channel's newest command: lower is + /// idle longer. + pub(crate) fn last_published(&self) -> u64 { + self.last_published + } } impl Queue { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d94e344c6f713..44770b0cb3041 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -155,12 +155,43 @@ impl Registry { Rebind::Fresh }); } - // Physical channels are a device budget, not a permanent entitlement - // of each open logical queue. Only exact idle retirement can yield one. - for queue in self.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; + // A binding persists for its logical queue's lifetime while physical + // QIDs allow: take a channel of a closed owner, then a fresh QID, and + // only then the least recently idle channel of one open owner. + if let Some(rebind) = self.bind_reusable(owner, context, required, admission)? { + return Ok(rebind); + } + if self.qids.compute_fits(self.admission.queues() as usize) { + return Ok(Rebind::Fresh); } + let teardown = &self.teardown; + let idle = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.shareable()) + .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) + .min_by_key(|queue| (!queue.pool_fits(required), queue.last_published())) + .map(|queue| queue.qid()); + let Some(qid) = idle else { return Err(ENOSPC); }; + admission.check()?; + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)?; + if !queue.share_idle()? { + return Err(ENOSPC); + } + Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) + } + + /// Binds `owner` to a reusable channel whose last owner released it, or + /// names a vacant one to re-arm. None if no channel is reusable. + fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; for queue in self @@ -183,9 +214,9 @@ impl Registry { } if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; - return Ok(Rebind::Done); + return Ok(Some(Rebind::Done)); } - Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) + Ok(vacant.map(Rebind::Rearm)) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -352,9 +383,10 @@ impl Registry { .ok_or_else(|| self.compute_admission_error( crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; + let queues = self.admission.queues() as usize; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .reserve_for(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID), queues) .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { @@ -491,6 +523,11 @@ impl Backend { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; + // Installed compute channels narrow render's QIDs: every open + // logical queue keeps a dedicated render pair within reach. + if !firmware.queues.qids.render_fits(firmware.queues.admission.queues() as usize) { + return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); + } match firmware.uat.new_execution_context(vm) { Ok(execution) => { break ( @@ -1861,8 +1898,9 @@ impl Registry { self.compute.iter().flatten().count(), self.compute.len(), self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; - writeln!(f, "compute_channels used={} total={}", - self.compute.iter().flatten().count(), super::kick::COMPUTE_CHANNELS)?; + writeln!(f, "compute_channels used={} fresh_fits={}", + self.compute.iter().flatten().count(), + u8::from(self.qids.compute_fits(self.admission.queues() as usize)))?; writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { queue.write_capacity(f)?; @@ -1893,10 +1931,6 @@ impl Backend { if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { return Ok(()); } - for queue in firmware.queues.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; - } // Consumption, not publication or an elapsed timeout, frees backing. firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; // Pool eviction addresses backing allocation pressure, not channel @@ -1937,12 +1971,17 @@ mod shared_capacity_tests { ) .unwrap(); let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - // Physical compute channels are device-owned and bounded independently - // of how many logical clients have queued unsignalled work. - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + let mut slots = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..crate::mmu::MAX_EXECUTION_CONTEXTS { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // With every execution context open, compute keeps only the channels + // that leave each logical queue a dedicated render pair. + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + assert!(registry.qids.render_fits(slots.len())); for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); @@ -1994,11 +2033,31 @@ mod shared_capacity_tests { leases.push(lease, GFP_KERNEL).unwrap(); } } - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert_eq!(leases.len(), 36); } + + #[test] + fn compute_grows_with_free_qids_and_narrows_queue_admission() { + let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); + for _ in 0..20 { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // Sixteen open compute clients each keep their own channel. + for owner in 1..=16u64 { + registry.reserve_compute(owner).unwrap(); + } + // Growth stops where open queues plus headroom would lose their pairs. + let mut owner = 17; + while registry.reserve_compute(owner).is_ok() { + owner += 1; + } + assert!(registry.qids.render_fits(slots.len() + 24)); + assert!(!registry.qids.render_fits(crate::mmu::MAX_EXECUTION_CONTEXTS as usize)); + } } impl Backend { From b141ca9b16c33abb26238782cd7c0b751b5f52fe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 21:40:28 -0400 Subject: [PATCH 13/64] drm/asahi: order same-VM render attachment overlaps in firmware A render whose attachment spans overlapped a live render of another logical queue in the same VM waited in admission, on the host, until that render retired. gpubench render.light queues renders every queue into one image, so all queues ran one render at a time: run_job blocked for (N-1) renders and the 2/4/8-queue rows stayed at one serialized step (174/201/217 us/submit). Order such overlaps at publication instead. The device mutex already orders publications; the later render's fragment kick now waits in firmware for the newest published overlapping fragment kick of every other pair, so its tiling kick stays concurrent and no host wait or scheduler stall is involved. Published QIDs keep their kick rings, so these stamps only advance. An admission bound on other pairs of the VM keeps the barrier count within its fixed room. Cross-VM overlaps through shared objects keep the host wait, so one VM's fault can never hold another VM's kick in firmware. render.light queues d2/d4/d8: 173.8/201.3/216.9 -> 81.4/66.9/62.1 us/submit. Other rows unchanged; the mild stress gate (CTS compute/sync, cross-queue order/data, slow-read pressure, mixed churn) passes. (cherry picked from commit 809d95cfdd685b6f94570cd313b1723c7f01f466) --- drivers/gpu/drm/asahi/g17/completion.rs | 7 ++ drivers/gpu/drm/asahi/g17/hazard.rs | 46 ++++++++++- drivers/gpu/drm/asahi/g17/queue/admission.rs | 78 ++++++++++++++++++- drivers/gpu/drm/asahi/g17/queue/render.rs | 17 +++- .../drm/asahi/g17/runtime/render/submit.rs | 10 ++- 5 files changed, 150 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 5936bcb1b933a..d692a98acea8c 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -547,6 +547,13 @@ impl Completion { .map(super::queue::admission::RenderLease::slot) } + /// Called under the device mutex once the render's kicks are committed. + pub(crate) fn note_render_publication(&self, fragment: super::fw::kick::KickDependency) { + if let Some(lease) = self.render.lock().as_ref() { + lease.note_publication(fragment); + } + } + /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs index 20daaab85b832..4d9e99d9a9cc0 100644 --- a/drivers/gpu/drm/asahi/g17/hazard.rs +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -3,11 +3,21 @@ //! Attachment overlap ordering across physical render pairs and shared images. //! Tile stores can overlap even when the application's pixel regions do not. //! Keep independent attachments concurrent; order only conflicting live spans. +//! +//! Within one VM, a render of another logical owner is ordered at publication: +//! the device mutex totally orders publications, and the later render's +//! fragment kick waits in firmware for the newest overlapping fragment kick of +//! every other pair. Admission never waits for such an overlap. Only a +//! cross-VM overlap through a shared object waits on the host for retirement, +//! so that one VM's fault can never hold another VM's kick in firmware. use super::job::Order; const MAX_SPANS: usize = 24; pub(crate) const WAIT_MS: u32 = 1000; +/// Firmware barriers one render publication may add for same-VM overlaps, +/// one per other pair. +pub(crate) const PUBLICATION_BARRIERS: usize = 7; #[derive(Clone, Copy)] struct Span { @@ -56,7 +66,12 @@ impl Footprint { self.shared |= self.count != before; } - fn overlaps(&self, other: &Self) -> bool { + /// Whether a publication orders an overlap with `other` in firmware. + pub(crate) fn same_vm(&self, other: &Self) -> bool { + self.vm == other.vm + } + + pub(crate) fn overlaps(&self, other: &Self) -> bool { if self.vm != other.vm && !(self.shared && other.shared) { return false; } @@ -86,6 +101,13 @@ pub(crate) struct Live<'a> { /// A queue prefix already orders covered predecessors. Pending arrivals order /// different owners fairly without imposing order on unrelated attachments. +/// +/// A same-VM overlap with another owner's admitted render needs no wait; its +/// publication carries one barrier per other pair. That count stays within +/// [`PUBLICATION_BARRIERS`] because such a render is admitted only while fewer +/// than `PUBLICATION_BARRIERS + 1` other pairs of its VM hold admitted renders: +/// of any set of renders overlapping one publication, the last admitted saw all +/// the others live. pub(crate) fn plan<'a>( owner: u64, prefix: Option, @@ -94,10 +116,29 @@ pub(crate) fn plan<'a>( live: impl IntoIterator>, ) -> Plan { let mut selected = None; + let mut ordered = false; + let mut pairs = [0u8; PUBLICATION_BARRIERS]; + let mut count = 0; + let mut crowded = false; for entry in live { + let foreign = entry.owner != owner && footprint.same_vm(entry.footprint); + if let Some(slot) = entry.slot.filter(|_| foreign) { + if !pairs[..count].contains(&slot) { + if count == PUBLICATION_BARRIERS { + crowded = true; + } else { + pairs[count] = slot; + count += 1; + } + } + } if entry.quarantined || !footprint.overlaps(entry.footprint) { continue; } + if foreign { + ordered |= entry.slot.is_some(); + continue; + } let Some(slot) = entry.slot else { if entry.owner != owner && ticket.is_none_or(|mine| entry.ticket < mine) { return Plan::Wait; @@ -116,5 +157,8 @@ pub(crate) fn plan<'a>( Some(_) => return Plan::Wait, } } + if ordered && crowded { + return Plan::Wait; + } selected.map_or(Plan::Any, Plan::Pair) } diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index ebdc79d2b2dd3..7533d345cfa6d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -9,6 +9,7 @@ //! a graph or waiting for GPU work. Capacity waiters sleep on the release event. use crate::g17::{ + fw::kick::KickDependency, hazard::{self, Footprint, Plan}, job::Packet, }; @@ -43,6 +44,9 @@ struct InFlight { slot: Option, ticket: u64, footprint: Arc, + /// Fragment kick of the published render; a later same-VM overlap of + /// another pair waits for it in firmware. + fragment: Option, } const QUARANTINE_WAIT_MS: u32 = 1000; @@ -332,6 +336,7 @@ impl State { slot, ticket, footprint: request.footprint.clone(), + fragment: None, }, GFP_KERNEL, ) @@ -339,6 +344,12 @@ impl State { Some((owner, request.sequence)) } + fn position(&self, slot: u8, (owner, sequence): (u64, u64)) -> Option { + self.footprints.iter().position(|entry| { + entry.slot == Some(slot) && entry.owner == owner && entry.sequence == sequence + }) + } + fn forget_waiter(&mut self, ticket: Option) { if let Some(ticket) = ticket { self.footprints @@ -705,13 +716,72 @@ impl RenderLease { self.rehabilitate_after_processor_stop(); } } +impl RenderLease { + /// Called under the device mutex once the render's kicks are committed, + /// including a replay, which supersedes the earlier kick. + pub(crate) fn note_publication(&self, fragment: KickDependency) { + let Some(record) = self.record else { + return; + }; + let mut state = self.pool.state.lock(); + if let Some(index) = state.position(self.slot, record) { + state.footprints[index].fragment = Some(fragment); + } + } +} + +impl Pool { + /// Under the device mutex, which orders every publication: the newest + /// published fragment kick of each other pair whose live same-VM render + /// overlaps `footprint`. Quarantined pairs are skipped as in admission. + pub(crate) fn publication_barriers( + &self, + owner: u64, + slot: u8, + footprint: &Footprint, + barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], + ) -> Result { + let state = self.state.lock(); + let mut slots = [0u8; hazard::PUBLICATION_BARRIERS]; + let mut count = 0; + for entry in state.footprints.iter() { + let (Some(other), Some(fragment)) = (entry.slot, entry.fragment) else { + continue; + }; + if other == slot + || entry.owner == owner + || state.quarantined & (1u64 << other) != 0 + || !footprint.same_vm(&entry.footprint) + || !footprint.overlaps(&entry.footprint) + { + continue; + } + match slots[..count].iter().position(|known| *known == other) { + // One fragment ring completes in kick order; same QID, so the + // word orders by timestamp. + Some(index) => { + if fragment.word() > barriers[index].word() { + barriers[index] = fragment; + } + } + // Admission bounds the pairs; see hazard::plan. + None if count == hazard::PUBLICATION_BARRIERS => return Err(EBUSY), + None => { + slots[count] = other; + barriers[count] = fragment; + count += 1; + } + } + } + Ok(count) + } +} + impl Drop for RenderLease { fn drop(&mut self) { let mut state = self.pool.state.lock(); - if let Some((owner, sequence)) = self.record { - if let Some(index) = state.footprints.iter().position(|entry| { - entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence - }) { + if let Some(record) = self.record { + if let Some(index) = state.position(self.slot, record) { state.footprints.swap_remove(index); } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index fb6c77fbdf04f..dfe3426eb6c24 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -46,6 +46,9 @@ pub(crate) struct RenderDependencies { pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, + /// Fragment kicks of other pairs' overlapping same-VM renders. + pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], + pub(crate) hazard_count: usize, } impl RenderDependencies { @@ -55,6 +58,8 @@ impl RenderDependencies { tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, + hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], + hazard_count: 0, }; pub(crate) fn render(&self) -> &[KickDependency] { &self.render[..self.render_count] @@ -484,7 +489,7 @@ impl Pair { struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, - fragment: [KickDependency; 3], + fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], fragment_count: usize, } @@ -495,7 +500,7 @@ fn render_barriers( ) -> RenderBarriers { let mut tiling_barriers = [KickDependency::ZERO; 3]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; let mut fragment_count = 1; for dependency in dependencies.render() { if fragment_stage { @@ -521,6 +526,11 @@ fn render_barriers( tiling_barriers[tiling_count] = compute; tiling_count += 1; } + // Only tile stores overlap: the tiling kick stays concurrent. + for hazard in &dependencies.hazards[..dependencies.hazard_count] { + fragment_barriers[fragment_count] = *hazard; + fragment_count += 1; + } RenderBarriers { tiling: tiling_barriers, tiling_count, @@ -840,6 +850,9 @@ impl Pair { } }; self.ordinal = next_ordinal; + if let Some(fragment) = KickDependency::new(qids[1], kicks[1]) { + packet.completion.note_render_publication(fragment); + } self.cursors = windows.map(|window| window.target()); for stage in 0..2 { if present[stage] { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index e7d0cf1168c98..663e62e8f86fc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -462,7 +462,15 @@ impl crate::g17::Firmware { { return Err(EBUSY); } - let dependencies = self.queues.render_dependencies(owner, packet, state)?; + let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.queues.admission.publication_barriers( + owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; // An owner change taken from a live owner which closed meanwhile: its // first command waits until that scheduler's release has been consumed. From a475f4f9f1f721b6636a4147dacbcf9f6224763b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:01:13 -0400 Subject: [PATCH 14/64] drm/asahi: map G17 compute command records cached for the firmware Each compute command writes a queue configuration record, an optional predecessor and completion-wait records, and an announcement; the firmware read all of them through an uncached mapping. Render keeps its equivalent configuration records cached and only its announcements uncached. Move the configuration, predecessor and completion-wait records into a retained pool object with a firmware-cached mapping at the same per-slot offsets; announcements stay uncached. compute.light d1 161.0 -> 152.2 us, d64 24.3 -> 23.1 us/submit, queue-to-queue hop 54.8 -> 51.2 us; compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit b3d5b9d6294bb46d63778264205da9d3b750651e) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 69841e5cd2aeb..625f65e6d6772 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -123,7 +123,11 @@ struct Graph { graph: KernelObject, context_low: KernelObject, context_high: KernelObject, + /// Announcements, which stay uncached for the firmware. records: KernelObject, + /// Configuration, predecessor and completion-wait records at the same + /// offsets, which the firmware reads through a cached mapping. + cached_records: PooledObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, /// Host words of the pointer block in this firmware build. @@ -171,6 +175,14 @@ impl Graph { let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + let cached_records = objects.allocate( + alloc, + Family::QueueConfig, + RECORD_SLOTS * RECORD_STRIDE, + RECORD_SLOTS * RECORD_STRIDE, + prot, + cpu, + )?; Ok(Self { ring: Some(ring), preempt, @@ -181,6 +193,7 @@ impl Graph { context_low, context_high, records, + cached_records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, pointers, @@ -1729,7 +1742,12 @@ impl Graph { if slot >= RECORD_SLOTS { return Err(EINVAL); } - (&mut self.records, slot * RECORD_STRIDE + offset) + let object: &mut KernelObject = if offset == ANNOUNCE { + &mut self.records + } else { + &mut self.cached_records + }; + (object, slot * RECORD_STRIDE + offset) } else { (&mut self.graph, 0x6800 + offset) }; From ca71e4449b4a23d5f4e0149db76cf5198bfa7e5f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:03:59 -0400 Subject: [PATCH 15/64] drm/asahi: stop the G17 compute completion scan once every record is found Every event pass with an unobserved compute command walked all 2048 records of the compute completion ring under the device mutex to find the few it needed; that walk was half of the event worker's CPU time. Resume each scan after the last record the previous scan needed and stop as soon as every requested identity has an observation. A scan that finds nothing still wraps the whole ring once, as before. scan_compute leaves the top of the 16-process profile; benchmark rows are unchanged within noise (they are not bound by the scan). Compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit 67a49c9c40b31b96a7834c8c185dc20e0bac1fa2) --- drivers/gpu/drm/asahi/g17/initdata.rs | 21 +++++++++++++------ .../drm/asahi/g17/queue/compute/retirement.rs | 12 +++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 15 +++++++++++-- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 049c93b6a466a..b9a8f326d970d 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -679,19 +679,28 @@ impl InitData { /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. + /// Visits ring slots from `start`, wrapping once, until `record` reports + /// that nothing more is wanted. Returns the slot after the last one visited + /// when the scan stopped early, or `start` after a complete wrap. pub(super) fn scan_compute_completions( &self, - mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { + start: usize, + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + ) -> Result { let object = self.object(self.completion_rings[0])?; + let slots = (abi::COMPLETION_RING_SIZE / 0x40).min(0x800); + let start = start % slots; fence(Ordering::Acquire); - for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - record(ComputeCompletion { + for step in 0..slots { + let slot = (start + step) % slots; + if record(ComputeCompletion { object, offset: slot * 0x40, - })?; + })? { + return Ok((slot + 1) % slots); + } } - Ok(()) + Ok(start) } fn object(&self, index: usize) -> Result<&KernelObject> { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index e938ec566dbac..655e2921f2582 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -174,6 +174,8 @@ pub(crate) struct Batch { requests: KVVec, slots: [(usize, usize); QIDS], open: Option, + /// Requests with no observation yet; the scan stops when this reaches zero. + unobserved: usize, } impl Batch { @@ -182,12 +184,18 @@ impl Batch { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, slots: [(0, 0); QIDS], open: None, + unobserved: 0, }) } pub(crate) fn clear(&mut self) { self.requests.clear(); self.slots.fill((0, 0)); self.open = None; + self.unobserved = 0; + } + /// True once every requested identity has an observation. + pub(crate) fn complete(&self) -> bool { + self.unobserved == 0 } pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { @@ -214,6 +222,7 @@ impl Batch { .map_err(|_| ENOSPC)?; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); + self.unobserved += 1; Ok(()) } /// Capture the control word once, then read only a requested identity's payload. @@ -247,6 +256,9 @@ impl Batch { { return Ok(()); } + if request.observed.is_none() { + self.unobserved -= 1; + } request.observed = Some(Observation { timestamps }); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 44770b0cb3041..745cd62cae595 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -86,6 +86,8 @@ pub(super) struct Registry { compute_scheduler_building: bool, replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, + /// Completion-ring slot where the next compute scan starts. + completion_cursor: usize, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -121,6 +123,7 @@ impl Registry { compute_scheduler_building: false, replay_outer_wait: None, observations: compute::retirement::Batch::new()?, + completion_cursor: 0, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) @@ -1403,9 +1406,17 @@ impl super::Firmware { { requested |= queue.request_observations(&mut registry.observations, masks)?; } + // Completion records are appended in order: resume after the last + // record a previous scan needed and stop once every request is found. if requested { - self.init - .scan_compute_completions(|record| registry.observations.fold(record))?; + let observations = &mut registry.observations; + registry.completion_cursor = self.init.scan_compute_completions( + registry.completion_cursor, + |record| { + observations.fold(record)?; + Ok(observations.complete()) + }, + )?; } let view = qos::View::new(self.init.qos()?)?; for queue in registry From 1cdac9ba0dddec96f59eb02a5067ee343aae43f4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:06:56 -0400 Subject: [PATCH 16/64] drm/asahi: publish G17 compute into a bound queue without the device mutex Every compute publication took the device mutex, which the event worker also holds for each service pass, and repeated device-wide work on every submit (replay scan, render-prefix scan, binding checks). Give each physical compute queue its own lock (a cell the registry owns; device-mutex holders lock cells one at a time) and move the device-wide state a publication needs into a per-session port that is reached under a preparation lease: compute work-ring slot reservations, QoS accounting under its own spinlock, the recovery generation, the full-effort flag and the doorbell, whose RTKit borrows are serialized by a mailbox lock. The steady-state path (bound, installed, no input sources, no render prefix) publishes from the submit ioctl or run_job with only the queue's lock and the port's small locks; everything else falls back to the device-mutex path, which now reserves work-ring slots through the same port. A command published while the event worker is scanning is not part of that scan's completion snapshot and is left unobserved until the next one. gpubench on Neo (us/submit, vs matched baseline): compute.light d1 166.3 -> 156.2, d64 27.4 -> 25.8, procs d16 33.4 -> 33.0 (firmware per command time now bounds these rows). Stress gate passes. (cherry picked from commit 8fc91b2117de925e362342a842b0d0befbba5312) --- drivers/gpu/drm/asahi/g17/context.rs | 4 + drivers/gpu/drm/asahi/g17/initdata.rs | 63 ++- drivers/gpu/drm/asahi/g17/kick.rs | 9 - drivers/gpu/drm/asahi/g17/mod.rs | 112 ++++- drivers/gpu/drm/asahi/g17/port.rs | 267 ++++++++++++ drivers/gpu/drm/asahi/g17/power.rs | 2 +- drivers/gpu/drm/asahi/g17/qos.rs | 19 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 5 + .../drm/asahi/g17/queue/compute/retirement.rs | 7 +- drivers/gpu/drm/asahi/g17/runtime.rs | 386 ++++++++++++++---- .../drm/asahi/g17/runtime/compute_control.rs | 13 +- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 28 +- .../drm/asahi/g17/runtime/compute_replay.rs | 22 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 12 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 21 +- drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 +- 16 files changed, 784 insertions(+), 188 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/port.rs diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 1818fd8c6922d..564d761d37f97 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -214,6 +214,10 @@ impl Context { self.render_seen.load(Ordering::Acquire) || self.compute_binding_hint.load(Ordering::Relaxed) } + /// This logical queue has submitted render work, so its compute may have a render prefix. + pub(crate) fn render_seen(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + } pub(crate) fn note_render(&self) { if !self.render_seen.load(Ordering::Relaxed) { self.render_seen.store(true, Ordering::Release); diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index b9a8f326d970d..6d0b2f9951956 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -129,14 +129,6 @@ impl WorkPublication<'_> { ); } } - - fn commit(self) -> u32 { - self.copy_record(); - fence(Ordering::SeqCst); - self.producer.store(self.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - self.next - } } /// Session-long storage and mappings. Aliases are released before their backing objects. @@ -516,7 +508,7 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - 2 => DataMaster::Compute, + // Compute publications reserve their slots through the port. _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; @@ -569,16 +561,6 @@ impl InitData { Ok(values) } - /// Inner queue records must be visible before this outer publication. The returned - /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work( - &self, - priority: u8, - record: &super::fw::channels::WorkSlot, - ) -> Result { - Ok(self.work_publication(priority, record)?.commit()) - } - /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -677,6 +659,49 @@ impl InitData { Ok(()) } + /// Raw views of the memory publishers share without the device mutex: every compute work + /// ring, the QoS table and the recovery state word. They stay valid for this session. + pub(super) fn port_views( + &self, + mailbox: *const super::Mailbox, + ) -> Result { + use super::fw::{channels, queue::{DataMaster, Policy}}; + let ring = |priority: u8| -> Result { + let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?; + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); + let state = self.object(self.cluster)?; + let word = |offset: usize| -> Result<*const AtomicU32> { + Ok(state.word(base + offset)?) + }; + Ok(super::port::RingView { + slots: self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel), + channels::WORK_RING_SLOTS as usize + * core::mem::size_of::(), + )?, + consumers: [ + word(abi::ChannelState::CONSUMER)?, + word(abi::ChannelState::UNK_10)?, + ], + producer: word(abi::ChannelState::PRODUCER)?, + }) + }; + let mut compute = [ring(0)?; Policy::PRIORITIES as usize]; + for (priority, view) in compute.iter_mut().enumerate().skip(1) { + *view = ring(priority as u8)?; + } + Ok(super::port::Views { + compute, + qos: self + .qos()? + .pointer(0, core::mem::size_of::())?, + recovery: self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, + mailbox, + }) + } + /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. /// Visits ring slots from `start`, wrapping once, until `record` reports diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 87a90516d1b42..f3183bb005df6 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -87,7 +87,6 @@ struct Identity { pub(crate) struct QueueIds { entries: [Option; QID_COUNT], initial_render: [Option; 2], - publication: u64, leased: usize, /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). floor: usize, @@ -98,7 +97,6 @@ impl QueueIds { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), initial_render: [None; 2], - publication: 0, leased: 0, // The reserve must keep render capacity above the floor. floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, @@ -239,13 +237,6 @@ impl QueueIds { Ok(id) } - /// Ordered under the device mutex; no atomic operation on the submission path. Compute and - /// render publications share it, so a watchdog can order work across both engines. - pub(crate) fn next_publication(&mut self) -> Result { - self.publication = self.publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.publication) - } - /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a85f68829018e..edcb2ae737d63 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -38,6 +38,7 @@ mod job; mod capacity; mod kick; mod object; +mod port; pub(crate) mod power; mod preparation; mod qos; @@ -395,13 +396,48 @@ impl rtkit::Operations for RoleOps { } } +/// The RTKit of one coprocessor. The mailbox driver serializes its own register writes; this +/// lock only serializes the exclusive borrows the RTKit API takes, so that publishers holding +/// different locks can ring the same doorbell. +#[pin_data] +struct Mailbox { + #[pin] + lock: Mutex<()>, + rtkit: core::cell::UnsafeCell>, +} + +// SAFETY: Every access to the RTKit goes through `with`, which holds the lock. +unsafe impl Sync for Mailbox {} + +impl Mailbox { + fn new(rtkit: rtkit::RtKit) -> Result>> { + KBox::pin_init( + pin_init!(Self { + lock <- new_mutex!((), "g17::Mailbox::lock"), + rtkit: core::cell::UnsafeCell::new(rtkit), + }), + GFP_KERNEL, + ) + } + + fn with(&self, f: impl FnOnce(Pin<&mut rtkit::RtKit>) -> R) -> R { + let _guard = self.lock.lock(); + // SAFETY: The lock excludes every other borrow of the RTKit. + f(Pin::new(unsafe { &mut *self.rtkit.get() })) + } + + fn send(&self, endpoint: u8, message: u64) -> Result { + self.with(|rtkit| rtkit.send_message(endpoint, message)) + } +} + /// A running firmware coprocessor. /// /// Dropping it stops the CPU before RTKit and the crash log buffer go away. struct Coprocessor { cpu: regs::CpuControl, running: bool, - rtkit: rtkit::RtKit, + rtkit: Pin>, state: Arc, _crash_buffer: Option, } @@ -456,7 +492,7 @@ impl Coprocessor { Ok(Coprocessor { cpu: regs::CpuControl::new(pdev, index)?, running: false, - rtkit: Self::receiver(dev, index, &state)?, + rtkit: Mailbox::new(Self::receiver(dev, index, &state)?)?, state, _crash_buffer: crash_buffer, }) @@ -485,19 +521,20 @@ impl Coprocessor { fn boot(&mut self) -> Result { self.cpu.start()?; self.running = true; - Pin::new(&mut self.rtkit).boot() + self.rtkit.with(|rtkit| rtkit.boot()) } /// Starts the firmware endpoints and hands the firmware its initialization root. An /// acknowledgment message is awaited here; one in status A is the caller's to poll. fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { - let mut rtkit = Pin::new(&mut self.rtkit); - rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; - rtkit.as_mut().start_endpoint(EP_DOORBELL)?; - rtkit.send_message( - EP_FIRMWARE, - MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), - )?; + self.rtkit.with(|mut rtkit| { + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + ) + })?; match ack { fw::variant::InitAck::Message => self.state.wait_for_ack(), fw::variant::InitAck::Status => Ok(()), @@ -505,8 +542,8 @@ impl Coprocessor { } /// Sends a host notification to the firmware. - fn notify(&mut self, msg: u64) -> Result { - Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) + fn notify(&self, msg: u64) -> Result { + self.rtkit.send(EP_DOORBELL, msg) } /// Stops the CPU. The firmware keeps its state, but no longer runs. @@ -573,6 +610,10 @@ struct Firmware { secondary: Option, primary: Coprocessor, bridge: Bridge, + /// Publication state shared with publishers that do not hold the device mutex. + port: Arc, + /// Fast-path submissions already accounted by the idle check. + seen_submissions: u64, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, @@ -647,12 +688,13 @@ impl Firmware { } fn apply_effort(&mut self, effort: u32) -> Result { - Self::apply_shared_effort(&self.init, &mut self.effort, effort) + Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) } fn apply_shared_effort( init: &initdata::InitData, state: &mut power::Effort, + port: &port::Port, effort: u32, ) -> Result { if state.applied() == effort && init.effort()? == effort { @@ -660,21 +702,35 @@ impl Firmware { } init.publish_effort(effort)?; state.did_apply(effort); + port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) } /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, - /// unretired work; quarantined work retaining mappings is excluded. + /// unretired work; quarantined work retaining mappings is excluded. Submissions published + /// without the device mutex count as activity, and a clear that races one is abandoned. fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - let effort = match self.effort.idle(now_ns, in_flight) { + let submissions = self.port.submissions(); + let submitted = submissions != self.seen_submissions; + self.seen_submissions = submissions; + if submitted { + self.effort.submit(); + } + let effort = match self.effort.idle(now_ns, in_flight || submitted) { power::IdleAction::None => None, power::IdleAction::Clear => { + self.port.set_effort_full(false); + if self.port.submissions() != submissions { + self.port.set_effort_full(true); + self.effort.submit(); + return Ok(Some(power::IDLE_HOLD_NS)); + } self.apply_effort(0)?; None } power::IdleAction::Recheck(delay) => Some(delay), }; - Ok(match (effort, self.idle_hold(now_ns, in_flight)) { + Ok(match (effort, self.idle_hold(now_ns, in_flight || submitted)) { (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), }) @@ -801,6 +857,8 @@ struct Shared { /// Counts `changed` notifications, so that a waiter which checked under /// the device mutex can tell whether it already missed one. changes: AtomicU64, + /// The port of the session in `state`; null before boot and once detachment begins. + port: core::sync::atomic::AtomicPtr, } impl_has_work! { @@ -900,6 +958,20 @@ impl Shared { self.changes.load(Ordering::Acquire) } + /// The session's port, valid while `lease` is held: shutdown and device loss join every + /// lease before they detach the session. + fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { + let _ = lease; + // SAFETY: The pointer is published before the session becomes reachable and cleared + // only after every lease has been joined, so it names a live port for 'a. + unsafe { self.port.load(Ordering::Acquire).as_ref() } + } + + /// Called after joining every preparation lease, before the session is detached. + fn forget_port(&self) { + self.port.store(core::ptr::null_mut(), Ordering::Release); + } + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { // Already queued if this fails, which is just as good. @@ -1081,6 +1153,7 @@ impl Gpu { state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), changes: AtomicU64::new(0), + port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, )?; @@ -1103,6 +1176,9 @@ impl Gpu { GFP_KERNEL, )?; let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + gpu.shared + .port + .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); *gpu.shared.state.lock() = Some(firmware); gpu.shared.queue_events(); Ok(gpu) @@ -1214,6 +1290,8 @@ impl Gpu { secondary, primary, bridge, + port: port::Port::new()?, + seen_submissions: 0, rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), @@ -1260,6 +1338,8 @@ impl Gpu { if let Some(root_entries) = root_entries { session.uat.confirm_firmware_root_entries(root_entries)?; } + let mailbox: *const Mailbox = &*session.primary.rtkit; + session.port.attach(session.init.port_views(mailbox)?)?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs new file mode 100644 index 0000000000000..63eab753632be --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-wide publication state that physical queues share without the device mutex. +//! +//! A firmware session owns its port, and the views it holds name that session's +//! firmware-shared memory. Code that does not hold the device mutex reaches the port only +//! through a preparation lease ([`super::Shared::port`]): shutdown and device loss join every +//! lease before they detach the session. Each piece below is atomic or has its own lock. +//! +//! Lock order: device mutex -> compute queue cell -> the locks of this port. + +use super::{ + fw::{ + channels::{WorkSlot, WORK_RING_SLOTS}, + queue::{DataMaster, Policy}, + }, + qos, +}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; +use kernel::{ + new_spinlock, + prelude::*, + sync::{Arc, SetOnce, SpinLock}, +}; + +const PRIORITIES: usize = Policy::PRIORITIES as usize; + +/// One priority class's compute work ring: its slot array and the cursors of its two firmware +/// consumers and of the host producer. +#[derive(Clone, Copy)] +pub(super) struct RingView { + pub(super) slots: *mut u8, + pub(super) consumers: [*const AtomicU32; 2], + pub(super) producer: *const AtomicU32, +} + +impl RingView { + fn cursors(&self) -> Result<[u32; 3]> { + // SAFETY: The views name 32-bit words of the session's retained firmware objects, which + // the port's owner keeps alive. Other agents only use single 32-bit accesses. + let values = unsafe { + [ + (*self.consumers[0]).load(Ordering::Relaxed), + (*self.consumers[1]).load(Ordering::Relaxed), + (*self.producer).load(Ordering::Relaxed), + ] + }; + if values.iter().any(|&value| value >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + Ok(values) + } + + /// Slots the host may still fill before reaching either consumer. + fn free(&self) -> Result { + let [first, second, producer] = self.cursors()?; + let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + Ok(room(first).min(room(second))) + } +} + +/// Raw views of the session's firmware-shared memory, fixed once the firmware is built. +pub(super) struct Views { + pub(super) compute: [RingView; PRIORITIES], + pub(super) qos: *mut u8, + pub(super) recovery: *const AtomicU32, + pub(super) mailbox: *const super::Mailbox, +} + +// SAFETY: The views only name memory and a mailbox owned by the firmware session, which outlives +// every user of the port. Accesses are atomic words or serialized by the port's locks. +unsafe impl Send for Views {} +// SAFETY: As above. +unsafe impl Sync for Views {} + +#[pin_data] +pub(super) struct Port { + views: SetOnce, + /// Compute work-ring slots reserved by publishers that have not exposed them yet. + #[pin] + rings: SpinLock<[u32; PRIORITIES]>, + #[pin] + qos: SpinLock, + /// The accepted recovery generation, mirrored for publishers without the device mutex. + generation: AtomicU64, + /// Device publication order shared by compute and render commands. + publications: AtomicU64, + /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. + effort_full: AtomicBool, + submissions: AtomicU64, +} + +/// A compute work-ring slot reserved before any command becomes visible. Dropping an unused +/// claim returns the slot. +pub(super) struct OuterClaim<'a> { + port: &'a Port, + priority: usize, + armed: bool, +} + +impl Drop for OuterClaim<'_> { + fn drop(&mut self) { + if self.armed { + let mut reserved = self.port.rings.lock(); + reserved[self.priority] = reserved[self.priority].saturating_sub(1); + } + } +} + +impl Port { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + views: SetOnce::new(), + rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), + qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), + generation: AtomicU64::new(0), + publications: AtomicU64::new(0), + effort_full: AtomicBool::new(false), + submissions: AtomicU64::new(0), + }), + GFP_KERNEL, + ) + } + + /// Called once while the session is built, before any publisher can see the port. + pub(super) fn attach(&self, views: Views) -> Result { + if self.views.populate(views) { + Ok(()) + } else { + Err(EBUSY) + } + } + + fn views(&self) -> Result<&Views> { + self.views.as_ref().ok_or(ENODEV) + } + + fn ring(&self, priority: u8) -> Result<(usize, RingView)> { + let index = usize::from(priority); + Ok((index, *self.views()?.compute.get(index).ok_or(EINVAL)?)) + } + + /// Reserves one slot of the compute work ring of `priority`. EAGAIN while firmware has not + /// consumed enough slots; only consumption, never a waiting publisher, frees one. + pub(super) fn reserve_compute(&self, priority: u8) -> Result> { + let (index, ring) = self.ring(priority)?; + let mut reserved = self.rings.lock(); + if ring.free()? <= reserved[index] { + return Err(EAGAIN); + } + reserved[index] += 1; + Ok(OuterClaim { + port: self, + priority: index, + armed: true, + }) + } + + /// Whether a compute publication at `priority` can reserve a slot now. + pub(super) fn compute_ready(&self, priority: u8) -> Result { + let (index, ring) = self.ring(priority)?; + let reserved = self.rings.lock(); + Ok(ring.free()? > reserved[index]) + } + + /// Exposes `record` in its reserved slot. Inner queue records must be visible first. Returns + /// the new producer, the retained consumer witness for this slot. + pub(super) fn publish_compute(&self, mut claim: OuterClaim<'_>, record: &WorkSlot) -> Result { + if record.data_master != DataMaster::Compute as u32 || !core::ptr::eq(claim.port, self) { + return Err(EINVAL); + } + let ring = self.views()?.compute[claim.priority]; + let mut reserved = self.rings.lock(); + let [first, second, producer] = ring.cursors()?; + let next = (producer + 1) % WORK_RING_SLOTS; + if next == first || next == second { + return Err(EIO); + } + // SAFETY: The ring view spans WORK_RING_SLOTS records, checked when the session built + // it; `producer` is in range. The reservation proves firmware does not own this slot and + // the ring lock excludes every other host writer. The record is a padding-free ABI value. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref(record).cast::(), + ring.slots.add(producer as usize * core::mem::size_of::()), + core::mem::size_of::(), + ); + } + fence(Ordering::SeqCst); + // SAFETY: As for `RingView::cursors`. + unsafe { (*ring.producer).store(next, Ordering::Relaxed) }; + fence(Ordering::SeqCst); + reserved[claim.priority] = reserved[claim.priority].saturating_sub(1); + claim.armed = false; + Ok(next) + } + + fn qos_view(&self) -> Result> { + // SAFETY: The session retains the QoS table for at least as long as this port. + unsafe { qos::View::from_raw(self.views()?.qos) } + } + + pub(super) fn qos_publish( + &self, + owner: qos::Owner, + scheduler: u64, + policy: Policy, + ) -> Result { + let view = self.qos_view()?; + self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock) + } + + pub(super) fn qos_cancel(&self, publication: qos::Publication) -> Result { + let view = self.qos_view()?; + self.qos.lock().cancel(&view, publication) + } + + pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result { + let view = self.qos_view()?; + self.qos.lock().complete(&view, owners) + } + + /// Rings the primary coprocessor's doorbell. + pub(super) fn notify(&self, message: u64) -> Result { + // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port. + let mailbox = unsafe { &*self.views()?.mailbox }; + mailbox.send(super::EP_DOORBELL, message) + } + + pub(super) fn recovery_state(&self) -> Result { + let word = self.views()?.recovery; + fence(Ordering::Acquire); + // SAFETY: As for `RingView::cursors`. + Ok(unsafe { (*word).load(Ordering::Relaxed) }) + } + + pub(super) fn generation(&self) -> u64 { + self.generation.load(Ordering::Acquire) + } + + /// Called with the device mutex held after the accepted generation changed, before + /// publication reopens. + pub(super) fn set_generation(&self, generation: u64) { + self.generation.store(generation, Ordering::Release); + } + + pub(super) fn next_publication(&self) -> u64 { + self.publications.fetch_add(1, Ordering::Relaxed) + 1 + } + + /// A publisher without the device mutex counts itself before it checks that full effort is + /// applied. The idle clear withdraws the flag before it rechecks the count, so either the + /// clear sees this submission or this submission sees the flag withdrawn. + pub(super) fn note_submission(&self) -> bool { + self.submissions.fetch_add(1, Ordering::SeqCst); + self.effort_full.load(Ordering::SeqCst) + } + + pub(super) fn submissions(&self) -> u64 { + self.submissions.load(Ordering::SeqCst) + } + + pub(super) fn set_effort_full(&self, full: bool) { + self.effort_full.store(full, Ordering::SeqCst); + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 7fef15084c50a..b0e21e70bbe18 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -6,7 +6,7 @@ use core::sync::atomic::AtomicU32; pub(super) const FULL_EFFORT: u32 = 0x10000; -const IDLE_HOLD_NS: u64 = 16_000_000; +pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; /// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after /// the last job. Not measured on other firmware; at or below it the hold stays off. diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index 9a100db4cffdf..4f6913ad5ee95 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -2,17 +2,14 @@ //! Shared QoS publication and exact-completion accounting for every engine. //! -//! The device publication lock serializes this host state. Publishing identity +//! The port's QoS lock serializes this host state. Publishing identity //! precedes submitted counters; retiring a final owner clears only its two row //! bytes. Cancellation balances host references without rewinding shared counts. -use super::{ - fw::{qos as fw, queue::Policy}, - object::KernelObject, -}; +use super::fw::{qos as fw, queue::Policy}; use crate::hw::t8140::qos as cfg; use core::{ - mem::{offset_of, size_of}, + mem::offset_of, sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, }; use kernel::prelude::*; @@ -30,15 +27,17 @@ pub(crate) struct View<'a> { times_and_counts: &'a [AtomicU64; 7], } impl<'a> View<'a> { - pub(crate) fn new(object: &'a KernelObject) -> Result { - let ptr = object.pointer(0, size_of::())?; + /// # Safety + /// + /// `ptr` must span a complete QoS table that stays mapped for `'a`. + pub(crate) unsafe fn from_raw(ptr: *mut u8) -> Result { if ptr as usize & 7 != 0 { return Err(EINVAL); } // SAFETY: pointer spans the complete layout, whose asserted offsets // and size bound every array below. Integer atomics have their integer - // representation and alignment. The retained object borrow keeps every - // shared reference alive; accesses use each field's fixed width. + // representation and alignment. The caller keeps every shared + // reference alive; accesses use each field's fixed width. Ok(unsafe { Self { rows: &*ptr.add(offset_of!(fw::Qos, rows)).cast(), diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 625f65e6d6772..a21d1c915d0c8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2351,6 +2351,11 @@ impl Queue { Ok(self.reusable()) } + /// The queue's first command has been published: its identity and configuration exist. + pub(crate) fn installed(&self) -> bool { + self.installed + } + /// An open client's idle channel that [`Self::share_idle`] can hand over. pub(crate) fn shareable(&self) -> bool { self.installed && self.owner.is_some() && self.retirement_ready diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 655e2921f2582..b268c1c40de00 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -262,16 +262,17 @@ impl Batch { request.observed = Some(Observation { timestamps }); Ok(()) } + /// A command published after this batch staged its queue's requests was not part of the + /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; - let request = self + Ok(self .requests .get_mut(start..start + count) .ok_or(EIO)? .iter_mut() .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) - .ok_or(EIO)?; - Ok(request.observed.take()) + .and_then(|request| request.observed.take())) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 745cd62cae595..237c92db48ce2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -69,13 +69,103 @@ pub(super) struct ComputeReservation { struct ComputeEntry { reservation: ComputeReservation, - queue: Option>, + queue: ComputeSlot, +} + +/// A physical compute queue behind its own lock. Publication into a bound queue takes only +/// this lock; device-mutex holders take it to observe or change the queue (device mutex -> +/// cell). A logical queue may keep the cell after the registry gave the queue up: the cell is +/// then empty, so the cached reference never extends any firmware-visible lifetime. +#[pin_data] +pub(super) struct ComputeCell { + #[pin] + queue: kernel::sync::Mutex>>, +} + +impl ComputeCell { + fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + queue <- kernel::new_mutex!(None, "g17::ComputeCell::queue"), + }), + GFP_KERNEL, + ) + } + + /// The installed queue, locked; None once the registry gave it up. + pub(super) fn lock(&self) -> Option> { + QueueGuard::new(self.queue.lock()) + } +} + +/// A locked installed queue. +pub(super) struct QueueGuard<'a> { + queue: core::ptr::NonNull, + _guard: kernel::sync::MutexGuard<'a, Option>>, +} + +impl<'a> QueueGuard<'a> { + fn new(mut guard: kernel::sync::MutexGuard<'a, Option>>) -> Option { + let queue = core::ptr::NonNull::from((*guard).as_deref_mut()?); + Some(Self { queue, _guard: guard }) + } +} + +impl core::ops::Deref for QueueGuard<'_> { + type Target = compute::Queue; + fn deref(&self) -> &compute::Queue { + // SAFETY: The boxed queue cannot be taken or dropped while the guard holds its lock. + unsafe { self.queue.as_ref() } + } +} + +impl core::ops::DerefMut for QueueGuard<'_> { + fn deref_mut(&mut self) -> &mut compute::Queue { + // SAFETY: As for `deref`; the lock makes this the only reference. + unsafe { self.queue.as_mut() } + } +} + +/// The registry's handle on one compute cell. Dropping it drops the queue in the cell, so a +/// cached cell never keeps a removed queue (or its QID's aliases) alive. +struct ComputeSlot(Arc); + +impl ComputeSlot { + fn new() -> Result { + Ok(Self(ComputeCell::new()?)) + } + fn as_deref(&self) -> Option> { + self.0.lock() + } + fn as_deref_mut(&mut self) -> Option> { + self.0.lock() + } + fn is_some(&self) -> bool { + self.0.queue.lock().is_some() + } + fn is_none(&self) -> bool { + !self.is_some() + } + /// Gives the queue up to the caller, which drops it outside the cell lock. + fn take(&mut self) -> Option> { + self.0.queue.lock().take() + } + fn put(&mut self, queue: Option>) { + let old = core::mem::replace(&mut *self.0.queue.lock(), queue); + drop(old); + } +} + +impl Drop for ComputeSlot { + fn drop(&mut self) { + let queue = self.0.queue.lock().take(); + drop(queue); + } } pub(super) struct Registry { pub(super) qids: kick::QueueIds, pub(super) buffers: BufferIds, - pub(super) accounting: qos::Accounting, pub(super) metrics: Arc, pub(super) qos_ids: Arc, pub(super) admission: Arc, @@ -106,7 +196,6 @@ impl Registry { qids.reserve_initial_render() }), buffers <- BufferIds::new(), - accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, @@ -129,7 +218,7 @@ impl Registry { }) } - pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { + pub(super) fn compute(&self, owner: u64) -> Option> { self.compute .iter() .flatten() @@ -137,12 +226,16 @@ impl Registry { .find(|queue| queue.owner() == Some(owner)) } - pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { - self.compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.owner() == Some(owner)) + pub(super) fn compute_mut(&mut self, owner: u64) -> Option> { + self.compute(owner) + } + + /// The cell of `owner`'s bound queue, which its publisher may lock without the device mutex. + pub(super) fn compute_cell(&self, owner: u64) -> Option> { + self.compute.iter().flatten().find_map(|entry| { + entry.queue.as_deref()?.owner().filter(|bound| *bound == owner)?; + Some(entry.queue.0.clone()) + }) } /// An installed idle queue keeps its original QID and backing. Mapping a @@ -175,18 +268,20 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.shareable()) .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) - .min_by_key(|queue| (!queue.pool_fits(required), queue.last_published())) - .map(|queue| queue.qid()); + .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) + .min_by_key(|(key, _)| *key) + .map(|(_, qid)| qid); let Some(qid) = idle else { return Err(ENOSPC); }; admission.check()?; - let queue = self + let shared = self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid) - .ok_or(EIO)?; - if !queue.share_idle()? { + .ok_or(EIO)? + .share_idle()?; + if !shared { return Err(ENOSPC); } Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) @@ -197,7 +292,7 @@ impl Registry { fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -230,7 +325,7 @@ impl Registry { pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -242,7 +337,7 @@ impl Registry { pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { let teardown = &self.teardown; - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -310,7 +405,7 @@ impl Registry { let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -340,7 +435,7 @@ impl Registry { let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); let mut freed = 0; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -371,7 +466,7 @@ impl Registry { || self.compute.iter().flatten().any(|entry| { entry .queue - .as_ref() + .as_deref() .map_or(entry.reservation.owner == owner, |queue| { queue.owner() == Some(owner) }) @@ -394,7 +489,7 @@ impl Registry { let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, - queue: None, + queue: ComputeSlot::new()?, }); Ok(reservation) } @@ -426,7 +521,7 @@ impl Registry { return Err(EIO); } let entry = self.reserved(reservation)?; - entry.queue = queue.take(); + entry.queue.put(queue.take()); Ok(()) } @@ -441,13 +536,14 @@ impl Registry { let Some(slot) = self.compute.iter().position(|entry| { entry .as_ref() - .and_then(|entry| entry.queue.as_ref()) + .and_then(|entry| entry.queue.as_deref()) .is_some_and(|queue| queue.owner() == Some(owner)) }) else { return Ok(()); }; let entry = self.compute[slot].as_mut().ok_or(EIO)?; - if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { + let unpublished = entry.queue.as_deref_mut().ok_or(EIO)?.release_owner()?; + if unpublished { self.qids.cancel_unpublished(entry.reservation.id)?; self.compute[slot] = None; } @@ -495,6 +591,7 @@ mod uninstalled_cleanup_tests { /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. +#[pin_data] pub(super) struct Backend { shared: Arc, pub(super) context: Arc, @@ -502,6 +599,11 @@ pub(super) struct Backend { backlog: core::sync::atomic::AtomicU32, compute_backlog: core::sync::atomic::AtomicU32, closed: core::sync::atomic::AtomicBool, + /// The cell of the compute queue this owner last published to under the device mutex, + /// with the shared compute objects mapped into its VM. Direct publication revalidates the + /// binding under the cell lock; a stale cell only sends the packet back to that path. + #[pin] + direct: kernel::sync::Mutex>>, } impl Backend { @@ -580,15 +682,16 @@ impl Backend { } drop(state); Ok(( - Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { shared: shared.clone(), context, owner, backlog: core::sync::atomic::AtomicU32::new(0), compute_backlog: core::sync::atomic::AtomicU32::new(0), closed: core::sync::atomic::AtomicBool::new(false), - }, + direct <- kernel::new_mutex!(None, "g17::Backend::direct"), + }), GFP_KERNEL, )?, slot, @@ -762,7 +865,7 @@ impl Backend { if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; + let mut queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } if growth.is_none() { let notify = queue.begin_growth(&firmware.init, required)?; @@ -793,8 +896,9 @@ impl Backend { if let Some(firmware) = state.as_deref_mut() { for index in 0..firmware.queues.compute.len() { let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; - let Some(queue) = entry.queue.as_deref_mut() else { continue; }; - if queue.abort_growth(token)? == Some(true) { + let Some(aborted) = entry.queue.as_deref_mut() + .map(|mut queue| queue.abort_growth(token)).transpose()? else { continue; }; + if aborted == Some(true) { // Keep the empty entry and QID reserved until all // graph aliases have been destroyed outside the lock. retired = entry.queue.take().map(|queue| (entry.reservation, queue)); @@ -1156,15 +1260,15 @@ impl DeferredBatch { struct ComputeHost<'a> { init: &'a super::initdata::InitData, - primary: &'a mut super::Coprocessor, + port: &'a super::port::Port, + outer: Option>, effort: &'a mut super::power::Effort, qids: &'a mut kick::QueueIds, - accounting: &'a mut qos::Accounting, generation: u64, } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - self.qids.next_publication() + Ok(self.port.next_publication()) } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -1181,30 +1285,66 @@ impl compute::Host for ComputeHost<'_> { scheduler: u64, policy: super::fw::queue::Policy, ) -> Result { - self.accounting.publish( - &qos::View::new(self.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + self.port.qos_publish(owner, scheduler, policy) } fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.accounting - .cancel(&qos::View::new(self.init.qos()?)?, publication) + self.port.qos_cancel(publication) } fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(priority, slot).map(|_| ()) + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { - self.primary.notify(message) + self.port.notify(message) } fn note_submission(&mut self) -> Result { self.effort.submit(); - super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) + super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) + } +} + +/// Publication into a bound, installed queue through the port alone. The caller checked the +/// recovery state, full effort and the queue's binding; first installation never comes here. +struct DirectHost<'a> { + port: &'a super::port::Port, + outer: Option>, + generation: u64, +} +impl compute::Host for DirectHost<'_> { + fn next_compute_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn prepare_compute_shared(&mut self) -> Result { + Err(EAGAIN) + } + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: super::fw::queue::Policy, + ) -> Result { + self.port.qos_publish(owner, scheduler, policy) + } + fn qos_cancel(&mut self, publication: qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn publish_qid(&mut self, _id: kick::Id) -> Result { + // An installed queue published its identity with its first command. + Ok(()) + } + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) + } + fn notify(&mut self, message: u64) -> Result { + self.port.notify(message) + } + fn note_submission(&mut self) -> Result { + Ok(()) } } @@ -1238,7 +1378,7 @@ impl super::Firmware { if source.submission.source_status()? { continue; } let found = self.queues.compute.iter_mut().flatten() .filter_map(|entry| entry.queue.as_deref_mut()) - .find_map(|queue| match queue.source_dependency(source, packet.completion.status(), + .find_map(|mut queue| match queue.source_dependency(source, packet.completion.status(), self.recovery.generation()) { Ok(None) => None, result => Some(result), @@ -1284,15 +1424,10 @@ impl super::Firmware { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // capacity check. The same device lock covers queue.publish below. - if !self - .init - .work_ready(packet.context.policy().priority(), DataMaster::Compute)? - { - return Err(EAGAIN); - } + // reservation; dropping it unused returns the work-ring slot. + let outer = self.port.reserve_compute(packet.context.policy().priority())?; let queues = &mut self.queues; - let queue = queues + let mut queue = queues .compute .iter_mut() .flatten() @@ -1310,10 +1445,10 @@ impl super::Firmware { } let mut host = ComputeHost { init: &self.init, - primary: &mut self.primary, + port: &self.port, + outer: Some(outer), effort: &mut self.effort, qids: &mut queues.qids, - accounting: &mut queues.accounting, generation: self.recovery.generation(), }; if native.is_empty() { @@ -1341,10 +1476,11 @@ impl Registry { fn release_compute_witness( &mut self, init: &super::initdata::InitData, + port: &super::port::Port, owner: u64, deferred: &mut DeferredBatch, ) -> Result { - let Some(queue) = self + let Some(mut queue) = self .compute .iter_mut() .flatten() @@ -1356,10 +1492,9 @@ impl Registry { if !queue.spared_pending() { return Ok(false); } - let view = qos::View::new(init.qos()?)?; queue.release_witnessed( super::recovery::Memory::recovery_state(init)?, - |owner| self.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; Ok(queue.spared_pending()) @@ -1371,7 +1506,7 @@ impl Registry { error: Error, deferred: &mut DeferredBatch, ) -> Result { - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -1418,8 +1553,8 @@ impl super::Firmware { }, )?; } - let view = qos::View::new(self.init.qos()?)?; - for queue in registry + let port = &self.port; + for mut queue in registry .compute .iter_mut() .flatten() @@ -1430,7 +1565,7 @@ impl super::Firmware { masks, generation, state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -1440,16 +1575,16 @@ impl super::Firmware { let terminal = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .and_then(compute::Queue::take_terminal); + .and_then(|mut queue| queue.take_terminal()); if let Some((status, error)) = terminal { status.record(error); self.settle_render_vm(&status, error, deferred)?; self.queues.fail_compute_vm(&status, error, deferred)?; } } - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() @@ -1457,7 +1592,7 @@ impl super::Firmware { { queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -1480,7 +1615,7 @@ impl Backend { firmware.replay_compute(&mut deferred)?; firmware .queues - .release_compute_witness(&firmware.init, self.owner, &mut deferred) + .release_compute_witness(&firmware.init, &firmware.port, self.owner, &mut deferred) })(); deferred.finish(); let pending = pending?; @@ -1500,6 +1635,78 @@ impl Backend { Err(EBUSY) } + /// Steady-state publication into this owner's bound channel without the device mutex: only + /// the channel's own lock and the port's small locks are taken, and nothing device-wide is + /// scanned. None leaves the packet untouched for the device-mutex path, which owns binding, + /// first installation, input sources, render prefixes, replay and recovery. + fn compute_direct( + &self, + packet: &Arc, + required: usize, + deferred: &mut DeferredBatch, + ) -> Option { + if packet.completion.sources().is_some() + || self.context.render_seen() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let cell = self.direct.lock().clone()?; + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire() { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let mut queue = cell.lock()?; + if !queue.installed() + || queue.spared_pending() + || !queue.pool_fits(required) + || queue.pending_switch_from().is_some() + || !matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)) + || !lease.is_current() + { + return None; + } + let outer = port.reserve_compute(packet.context.policy().priority()).ok()?; + let mut host = DirectHost { + port, + outer: Some(outer), + generation: port.generation(), + }; + match queue.publish(packet.clone(), &[], &mut host, &mut |event| deferred.push(event)) { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => Some(result), + } + } + + /// Called with the device mutex held after a successful publication: remembers the bound + /// channel for direct publication. + fn remember_direct(&self, registry: &Registry) { + let cell = registry.compute_cell(self.owner); + let mut direct = self.direct.lock(); + let current = match (&*direct, &cell) { + (Some(old), Some(new)) => Arc::ptr_eq(old, new), + (None, None) => true, + _ => false, + }; + if !current { + let old = core::mem::replace(&mut *direct, cell); + drop(direct); + drop(old); + } + } + /// Publishes a dependency-free compute command from its submit ioctl when /// its queue is already bound and nothing has to wait, as renders do. Any /// wait leaves the command, unpublished, to its scheduler. @@ -1523,7 +1730,10 @@ impl Backend { return true; } }; - let result = self.compute_inline(packet, &mut deferred); + let result = match self.compute_direct_packet(packet, &mut deferred) { + Some(result) => result, + None => self.compute_inline(packet, &mut deferred), + }; if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { deferred.finish(); packet.abandon_publication(); @@ -1536,6 +1746,21 @@ impl Backend { true } + fn compute_direct_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Option { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return Some(Err(EINVAL)); + }; + let required = match super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) { + Ok(required) => required, + Err(error) => return Some(Err(error)), + }; + self.compute_direct(packet, required, deferred) + } + /// compute_packet's publication section without its waits: every condition /// it would sleep or retry for returns FALLBACK instead. fn compute_inline( @@ -1582,7 +1807,12 @@ impl Backend { Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { Err(render::FALLBACK) } - result => result, + result => { + if result.is_ok() { + self.remember_direct(&firmware.queues); + } + result + } } } @@ -1617,6 +1847,9 @@ impl Backend { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if let Some(result) = self.compute_direct(packet, required, deferred) { + return result; + } // A bound queue needs none of ensure_compute's admission passes. Try // without them; the publication section below checks the same state. let mut bound = true; @@ -1730,6 +1963,9 @@ impl Backend { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); continue; } + if result.is_ok() { + self.remember_direct(&firmware.queues); + } return result; } } @@ -1741,7 +1977,7 @@ impl super::Firmware { /// or an allocation-dependent collector. fn finish_stopped_queues(&mut self) { self.stop_render(); - for queue in self + for mut queue in self .queues .compute .iter_mut() @@ -1797,6 +2033,7 @@ impl super::Shared { dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); return; } + self.forget_port(); let firmware = self.state.lock().take(); if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } } @@ -1810,6 +2047,7 @@ impl super::Shared { pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); let _stopping = self.stop.lock(); + self.forget_port(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { @@ -1849,7 +2087,7 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { - for queue in self + for mut queue in self .queues .compute .iter_mut() diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index c407d9576e9c4..4f519743aed53 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,11 +20,10 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute(self.owner) - .filter(|queue| queue.can_classify_timeout(packet)) else { - return Ok(Decision::Own); - }; - let Some(own) = queue.timeout_head() else { + // Each queue is locked on its own: release this one before visiting the others. + let Some(own) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) + .and_then(|queue| queue.timeout_head()) else { return Ok(Decision::Own); }; let computes = firmware.queues.compute.iter().flatten() @@ -64,7 +63,7 @@ impl Backend { return Ok(None); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute_mut(self.owner) else { + let Some(mut queue) = firmware.queues.compute_mut(self.owner) else { return Ok(None); }; if !queue.owns_packet(packet) { @@ -91,7 +90,7 @@ impl Backend { return Ok(()); }; firmware.scan_compute(None, &mut deferred)?; - if let Some(queue) = firmware.queues.compute_mut(self.owner) { + if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 984b6c1f643eb..69bc7550fc4e9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -19,7 +19,7 @@ impl super::Registry { /// batch cannot overflow a separate cache. No allocation or wait is needed. pub(super) fn acknowledge_compute_exit(&mut self, cookie: u64) { for entry in self.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { if queue.acknowledge_exit(cookie) { return; } @@ -31,12 +31,12 @@ impl super::Registry { impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { let recovery_closed = self.init.recovery_state()? == 0; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.settle_exit( recovery_closed, - &mut |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -48,7 +48,7 @@ impl Firmware { // Publication has already retained each kill. Retry only the doorbell; // failed attempts do not extend its bounded polling window. for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { let _ = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)); } } @@ -85,14 +85,16 @@ impl Firmware { if self.render_control_backpressured() { return Ok(()); } - let queue = self.queues.compute[index] - .as_ref() - .and_then(|entry| entry.queue.as_deref()) - .ok_or(EIO)?; - let Some(owner) = queue.exit_owner(context) else { - return Ok(()); + let (owner, target) = { + let queue = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .ok_or(EIO)?; + let Some(owner) = queue.exit_owner(context) else { + return Ok(()); + }; + (owner, queue.qid()) }; - let target = queue.qid(); let (mut own, mut foreign, slots) = self.queues.render.exit_scope(context); for entry in self.queues.compute.iter().flatten() { if let Some(queue) = entry.queue.as_deref() { @@ -130,7 +132,7 @@ impl Firmware { } // Publication is irrevocable even if its doorbell fails. Keep both // the killed graph and the held render identities until processor stop. - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index d71f410c76fb4..0a8ebc28c02e2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -3,7 +3,7 @@ //! Ordered republication of witnessed, unexecuted compute commands. use super::DeferredBatch; -use crate::g17::{completion::Deferred, qos, recovery, Firmware}; +use crate::g17::{completion::Deferred, recovery, Firmware}; use kernel::prelude::*; impl Firmware { @@ -31,12 +31,12 @@ impl Firmware { return Ok(()); } deferred.reserve_replays(commands)?; - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.prepare_replays( state, - |owner| self.queues.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -46,20 +46,14 @@ impl Firmware { if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { return Ok(()); } - let Some(queue) = self.queues.compute[index] + let Some((packet, owner)) = self.queues.compute[index] .as_ref() .and_then(|entry| entry.queue.as_deref()) + .and_then(|queue| Some((queue.replay_front()?, queue.owner()))) else { break; }; - let Some(packet) = queue.replay_front() else { - break; - }; - let owner = queue.owner(); - let outer_ready = self.init.work_ready( - packet.context.policy().priority(), - crate::g17::fw::queue::DataMaster::Compute, - )?; + let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -90,7 +84,7 @@ impl Firmware { let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4fd6e56741986..e46c820efa155 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -225,9 +225,9 @@ impl recovery::Queues for Service<'_> { impl Firmware { fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { let state = recovery::Memory::recovery_state(&self.init)?; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() @@ -235,7 +235,7 @@ impl Firmware { { queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -276,7 +276,11 @@ impl Firmware { }; let result = recovery::service(&mut service, || { shared.crashed.load(core::sync::atomic::Ordering::Acquire) - })?; + }); + // Publishers without the device mutex stamp their tickets with this + // mirror; they rejoin only after preparation admission reopens. + service.firmware.port.set_generation(service.firmware.recovery.generation()); + let result = result?; if let Some(error) = result.target_error { dev_warn!( shared.dev.as_ref(), diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index c657a45bdfa48..2877dea91d0ea 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -514,21 +514,10 @@ impl crate::g17::queue::render::Host for Host<'_> { scheduler: u64, policy: crate::g17::fw::queue::Policy, ) -> Result { - use crate::g17::qos; - self.firmware.queues.accounting.publish( - &qos::View::new(self.firmware.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + self.firmware.port.qos_publish(owner, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { - use crate::g17::qos; - self.firmware - .queues - .accounting - .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) + self.firmware.port.qos_cancel(publication) } /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration @@ -559,7 +548,7 @@ impl crate::g17::queue::render::Host for Host<'_> { self.firmware.queues.qids.publish(id) } fn next_publication(&mut self) -> Result { - self.firmware.queues.qids.next_publication() + Ok(self.firmware.port.next_publication()) } fn publish_outer_pair( &mut self, @@ -619,13 +608,11 @@ impl crate::g17::queue::render::Host for Host<'_> { generation: u64, buffer_id: u8, ) -> Result { - use crate::g17::qos; let queues = &mut self.firmware.queues; let next = queues.render.pools.completed(pool_id, generation)?; queues.buffers.can_release(buffer_id)?; - let view = qos::View::new(self.firmware.init.qos()?)?; if let Some(owners) = owners { - queues.accounting.complete(&view, &owners)?; + self.firmware.port.qos_complete(&owners)?; } queues.render.pools.commit_completed(pool_id, next); queues.buffers.release(buffer_id) diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index f8020e8962f84..aaa2e281c69ee 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -115,7 +115,7 @@ impl Host for Service<'_> { self.compute .iter() .flatten() - .filter_map(|entry| entry.queue.as_ref()) + .filter_map(|entry| entry.queue.as_deref()) .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { From 7242bf87efd2b3f4f8bf2a31fadb79b3e687f195 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:21:42 -0400 Subject: [PATCH 17/64] drm/asahi: send a G17 compute queue configuration only when it changes Render emits its queue configuration record only for an install, a context or QoS update, or when the configuration differs from the last one the firmware consumed in the current recovery epoch. Compute appended the record to every command, so the firmware reapplied unchanged queue state each time. Use the same rule for compute, caching the last consumed configuration with its epoch and forgetting it when the item ring is reset. compute.light d2 61.9 -> 51.5, d8 25.9 -> 24.3, d64 23.9 -> 22.8 us/submit; queue-to-queue hop 51.1 -> 49.6 us. Compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit 0e59b1f92effb58c87f705907959de45abe77623) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 36 +++++++++++++------ drivers/gpu/drm/asahi/g17/queue/render.rs | 2 +- .../drm/asahi/g17/queue/render/publication.rs | 2 +- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index a21d1c915d0c8..5e6f20a3da43a 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -545,6 +545,9 @@ pub(crate) struct Queue { /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, + /// Last configuration record the firmware consumed, and its recovery + /// epoch. A command repeats the record only when it differs. + config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, last_published: u64, @@ -609,6 +612,7 @@ impl Queue { installed: false, outer_started: 0, published_config: None, + config_cache: None, ordinal: 0, submitted: 0, last_published: 0, @@ -1436,6 +1440,7 @@ impl Queue { } fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; + self.config_cache = None; self.exit = None; self.quarantined = false; self.quarantine_error = None; @@ -1925,6 +1930,8 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; + let cached = self.config_cache.and_then(|(config, at)| (at == epoch).then_some(config)); + let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -1989,7 +1996,11 @@ impl Queue { let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } - let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; + let config_va = if send_config { + Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) + } else { + None + }; let cursors = self.graph.cursors()?; let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; for (index, wait) in native.iter().enumerate() { @@ -2023,7 +2034,7 @@ impl Queue { })?; producer = self .graph - .append(producer, config_va, || {}) + .append(producer, config_va.ok_or(EINVAL)?, || {}) .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; @@ -2060,14 +2071,16 @@ impl Queue { policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; + if let Some(config_va) = config_va { + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; + } } if first { prepared_kick.prepare_compute_install()?; @@ -2101,6 +2114,9 @@ impl Queue { self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); + if send_config { + self.config_cache = Some((config, epoch)); + } self.ordinal = ordinal; let (accepted, state) = host.epoch()?; if state != 0 || accepted != epoch { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index dfe3426eb6c24..e063ef3a047fc 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -7,7 +7,7 @@ pub(crate) use observation::Observed; mod dependencies; mod memory; pub(crate) mod parked; -mod publication; +pub(super) mod publication; pub(crate) mod retirement; mod warm; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 9f4cb05b6bede..c1985c72f6ec1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -19,7 +19,7 @@ fn identity(mut config: QueueConfig) -> QueueConfig { config.qos_update = 0; config } -pub(super) fn required(previous: Option, config: QueueConfig) -> bool { +pub(in crate::g17::queue) fn required(previous: Option, config: QueueConfig) -> bool { config.install != 0 || config.context_update != 0 || config.completion_seed_valid != 0 From 014605a3679581351913c3ab137653297f7a0cf7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:28:39 -0400 Subject: [PATCH 18/64] drm/asahi: add G17 resource wait domains and use them for queue creation Waits for shared capacity sampled a condition and slept 1 ms before checking again. Add availability domains: a waiter samples the domain's generation before its failed check and registers a fence against that sample (or retries when a release intervened); the release that provides the resource bumps the generation and signals only that domain's waiters. The fences can serve as scheduler prepare_job dependencies or be waited on directly. The preparation gate can carry a domain that its reopening, failure and removal signal. Logical-queue creation is the first user: a creator refused for want of a queue slot or an execution context held by a closing queue now waits for the returned slot or the reclaimed context instead of sleeping in 1 ms steps, under the same overall deadline. Tested on Neo as part of compute-wait2 (stress gate passes). (cherry picked from commit 79cf4be7732d549b0f5d1f903bed7effdb80a4fb) --- drivers/gpu/drm/asahi/g17/availability.rs | 107 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/mod.rs | 20 +++- drivers/gpu/drm/asahi/g17/preparation.rs | 28 ++++- drivers/gpu/drm/asahi/g17/queue/admission.rs | 13 +++ drivers/gpu/drm/asahi/g17/runtime.rs | 20 +++- 5 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/availability.rs diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs new file mode 100644 index 0000000000000..5b5d1d639a18e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Resource-specific scheduler waits. A caller samples a domain before a failed try, then +//! registers against that sample after rolling back unpublished claims; the release that frees +//! the resource bumps the generation and signals exactly the waiters of its own domain. +//! Successful acquisition never allocates a fence, and a release without waiters costs one +//! lock round trip. + +use super::fence::{self, CompletionFence}; +use kernel::{ + c_str, + dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; + +struct Entry { + fence: UserFence, + next: Option>, +} + +struct State { + generation: u64, + waiters: Option>, +} + +#[pin_data] +pub(crate) struct Availability { + #[pin] + state: Mutex, + fences: FenceContexts, +} + +impl Availability { + pub(crate) fn new() -> Result> { + Arc::pin_init( + try_pin_init!(Self { + state <- new_mutex!( + State { generation: 0, waiters: None }, + "g17::Availability" + ), + fences: FenceContexts::new(1, c_str!("asahi_resource"), kernel::static_lock_class!())?, + }), + GFP_KERNEL, + ) + } + + /// Sample before checking capacity under the resource's own synchronization. + pub(crate) fn observe(&self) -> u64 { + self.state.lock().generation + } + + /// A fence the next notification signals. None means capacity changed after the sample: + /// retry instead of waiting. Called with no resource lock held. The recheck after allocation + /// closes both the failed-check/registration and the fence-allocation/release races. A + /// waiter that goes away leaves its entry for the next notification to signal. + pub(crate) fn wait(&self, observed: u64) -> Result> { + if self.observe() != observed { + return Ok(None); + } + let fence = fence::independent(&self.fences)?; + let mut entry = KBox::new( + Entry { fence: fence.clone(), next: None }, + GFP_KERNEL, + )?; + let mut state = self.state.lock(); + if state.generation != observed { + return Ok(None); + } + entry.next = state.waiters.take(); + state.waiters = Some(entry); + Ok(Some(Fence::from_fence(&fence))) + } + + /// Blocks until a notification after `observed`, for at most `timeout_ms`. For callers + /// outside the scheduler that own no other wait. + pub(crate) fn block(&self, observed: u64, timeout_ms: u32, interruptible: bool) -> Result { + let Some(fence) = self.wait(observed)? else { return Ok(()); }; + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + interruptible, + kernel::time::msecs_to_jiffies(timeout_ms) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + Ok(()) + } + + /// Wakes every waiter of this domain. Call after dropping the resource's own lock; fence + /// callbacks run outside the domain lock. + pub(crate) fn notify(&self) { + let mut waiters = { + let mut state = self.state.lock(); + state.generation = state.generation.wrapping_add(1); + state.waiters.take() + }; + while let Some(mut entry) = waiters { + waiters = entry.next.take(); + entry.fence.signal(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index edcb2ae737d63..3b8a615d1c533 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -17,6 +17,7 @@ //! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state //! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. +mod availability; mod buffer; mod channel; mod command; @@ -830,6 +831,8 @@ struct Shared { #[pin] feed_work: Work, preparations: Arc, + /// Logical-queue creation waiting for a returned queue slot or execution context. + queues: Arc, /// Unbound compute publishers which can reuse a retired channel. compute_demand: AtomicU32, #[pin] @@ -1000,6 +1003,7 @@ impl Shared { // Context release returns its execution root off-lock. The next // device pass can now detach idle compute aliases and their QoS // leases, without depending on a new client's submission/event. + self.queues.notify(); self.queue_events(); } } @@ -1112,6 +1116,7 @@ impl Gpu { }; let max_frequency_khz = Self::max_frequency_khz(dev)?; let preparations = preparation::Gate::new()?; + let queues = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); @@ -1142,6 +1147,7 @@ impl Gpu { feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + queues, compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), @@ -1298,11 +1304,12 @@ impl Gpu { hold: power::Hold::new(), registration_power_release_pending: false, variant: shared.variant, - queues <- runtime::Registry::new( + queues <- runtime::Registry::with_waiters( shared.clusters, shared.deflake, shared.descriptor_flags, shared.variant, + Some(shared.queues.clone()), ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, @@ -1573,16 +1580,21 @@ impl gpu::Gpu for Gpu { let policy = scheduling_policy(priority)?; let admission_started = now_ns(); let (backend, slot) = loop { + let observed = self.shared.queues.observe(); match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, Err(error) if error == ENOSPC => { // Closed logical queues can still own execution/QoS IDs // until the scheduler release is consumed and reclaimed. // Backend construction has rolled back all temporary leases. - if now_ns().saturating_sub(admission_started) - >= job::TIMEOUT_MS as u64 * 1_000_000 { return Err(EBUSY); } + let waited = now_ns().saturating_sub(admission_started); + let limit = job::TIMEOUT_MS as u64 * 1_000_000; + if waited >= limit { return Err(EBUSY); } self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A returned queue slot or reclaimed context wakes this. + self.shared + .queues + .block(observed, (limit - waited).div_ceil(1_000_000) as u32, true)?; } result => break result?, } diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 0297c619b4f93..e014d4bbb6542 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -55,20 +55,36 @@ pub(super) struct Gate { wait: Mutex<()>, #[pin] changed: CondVar, + /// Scheduler admission waiting for publication to reopen. + available: Option>, } impl Gate { pub(super) fn new() -> Result> { + Self::with_waiters(None) + } + + /// A gate whose reopening, failure and removal also wake `available`. + pub(super) fn with_waiters( + available: Option>, + ) -> Result> { Arc::pin_init( pin_init!(Self { state: AtomicU64::new(EPOCH), wait <- new_mutex!((), "g17::Gate::wait"), changed <- new_condvar!("g17::Gate::changed"), + available, }), GFP_KERNEL, ) } + fn wake_waiters(&self) { + if let Some(available) = &self.available { + available.notify(); + } + } + pub(super) fn try_enter(self: &Arc) -> Result { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -168,14 +184,18 @@ impl Gate { pub(super) fn remove(&self) { self.invalidate(REMOVED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } pub(super) fn fail(&self) { self.invalidate(FAILED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } /// Runtime failure/recovery must not join an indefinitely stalled builder. @@ -199,7 +219,7 @@ impl Gate { /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. pub(super) fn reopen(&self) -> bool { - let _wait = self.wait.lock(); + let wait = self.wait.lock(); let mut previous = self.state.load(Ordering::Acquire); loop { if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { @@ -213,6 +233,8 @@ impl Gate { ) { Ok(_) => { self.changed.notify_all(); + drop(wait); + self.wake_waiters(); return true; } Err(current) => previous = current, diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 7533d345cfa6d..ff2b294b6f35c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -377,9 +377,18 @@ pub(crate) struct Pool { queues: AtomicU32, /// Logical queues admitted at most: each must be able to hold a render pair. max_queues: u32, + /// Logical-queue creation waiting for a returned slot. + waiters: Option>, } impl Pool { + #[cfg(CONFIG_KUNIT)] pub(crate) fn new(max_queues: u32) -> Result> { + Self::with_waiters(max_queues, None) + } + pub(crate) fn with_waiters( + max_queues: u32, + waiters: Option>, + ) -> Result> { if max_queues > MAX_EXECUTION_CONTEXTS { return Err(EINVAL); } @@ -389,6 +398,7 @@ impl Pool { released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), max_queues, + waiters, }), GFP_KERNEL, ) @@ -667,6 +677,9 @@ pub(crate) struct QueueSlot { impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); + if let Some(waiters) = &self.pool.waiters { + waiters.notify(); + } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 237c92db48ce2..88a3330155cac 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -185,11 +185,23 @@ pub(super) struct Registry { impl Registry { /// Initialize inside the firmware allocation instead of staging the whole /// registry on the probe stack. No additional allocation is required. + #[cfg(CONFIG_KUNIT)] pub(super) fn new( clusters: u32, deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + ) -> impl Init { + Self::with_waiters(clusters, deflake, descriptor_flags, variant, None) + } + + /// `queues` wakes logical-queue creation when a logical queue slot is returned. + pub(super) fn with_waiters( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, + queues: Option>, ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { @@ -198,7 +210,7 @@ impl Registry { buffers <- BufferIds::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, + admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { @@ -621,6 +633,7 @@ impl Backend { const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; let started = now_ns(); let (lease, uat, pool, qos, execution, slot) = loop { + let observed = shared.queues.observe(); let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -660,7 +673,10 @@ impl Backend { return Err(ERESTARTSYS); } shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A closing queue returns its context when its release + // is reclaimed; that reclaim wakes this creator. + let left = RETAINED_CONTEXT_WAIT_NS.saturating_sub(now_ns().saturating_sub(started)); + shared.queues.block(observed, left.div_ceil(1_000_000) as u32, true)?; } Err(error) => return Err(error), } From 64f717dcfd1e4aa5856882181f5c2ac040c5ce4e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:30:03 -0400 Subject: [PATCH 19/64] drm/asahi: wait for G17 compute capacity on the release that frees it Compute admission and publication retried with sleeps: 1 ms for an unpublished input source, a 20 us..1 ms backoff for a work state, up to 200 x 1 ms for recovery, 1 ms per ring backpressure retry, 200 x 1 ms for a spared drain witness, and 1 ms per growth and channel capacity retry. Use two availability domains for compute admission; the release that provides the resource signals only the waiters of its own domain: - channels: a queue becomes reusable, shareable or re-armable (checked on each transition in the retirement scan and pool service), a scheduler release is consumed, an owner closes, or an unpublished slot and QID are returned; - progress: any compute retirement, finished replays, pool release observation, and the preparation gate reopening after recovery. prepare_job now returns the channel, work-state or recovery fence a compute packet still lacks, so run_job finds its capacity ready, and a prepared packet claims its bound channel so the channel is not shared away before run_job. Retries that remain inside publication block on the same fences; the admission slice only bounds cancellation checks. Each event pass also rechecks the level while waiters exist, which covers capacity that appears through render (QIDs, work states, closed queues). gpubench on Neo (us/submit, matched baseline): unchanged steady state (procs d16 33.1, d64 25.9, d1 162); procs d2 93.2 -> 69.2. With the channel budget forced back to ten channels (diagnostic build), sixteen processes now share them at 51.9 us/submit instead of the old 92.0, with no stall. Stress gate passes, also under the forced shortage. (cherry picked from commit 25fb5800d97db7457f093ff41206047fbd9bddcc) --- drivers/gpu/drm/asahi/g17/availability.rs | 4 + drivers/gpu/drm/asahi/g17/job.rs | 4 +- drivers/gpu/drm/asahi/g17/mod.rs | 24 +- drivers/gpu/drm/asahi/g17/preparation.rs | 1 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 24 +- drivers/gpu/drm/asahi/g17/runtime.rs | 314 +++++++++++++++--- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 18 +- .../drm/asahi/g17/runtime/compute_replay.rs | 2 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 9 +- 9 files changed, 337 insertions(+), 63 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs index 5b5d1d639a18e..8014266f0b0ab 100644 --- a/drivers/gpu/drm/asahi/g17/availability.rs +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -51,6 +51,10 @@ impl Availability { self.state.lock().generation } + pub(crate) fn has_waiters(&self) -> bool { + self.state.lock().waiters.is_some() + } + /// A fence the next notification signals. None means capacity changed after the sample: /// retry instead of waiting. Called with no resource lock held. The recheck after allocation /// closes both the failed-check/registration and the fence-allocation/release races. A diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 4ad4f8e372764..ae401b06f2eb3 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -282,7 +282,9 @@ pub(crate) trait Backend: Send + Sync { fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. fn release_owner(&self); - fn prepare(&self, packet: &Packet) -> Option; + /// A fence to wait for before `publish`, signalled by the release of the capacity the + /// packet lacks. Called again once it signals. + fn prepare(&self, packet: &Arc) -> Option; fn publish(&self, packet: Arc) -> Result; fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 3b8a615d1c533..330f57ab59779 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -12,10 +12,14 @@ //! //! # Locking //! -//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all -//! firmware-shared state, including the bounded waits for the firmware that publication needs. -//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state -//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. +//! `Shared::state` is the device lock. It serializes binding, recovery, the event worker and all +//! firmware-shared state that has no narrower owner. Steady-state compute publication into a +//! bound queue takes only that queue's cell lock and the port's locks ([`port`]), reached under +//! a preparation lease. Completions are signalled after dropping the device lock. Lock order: +//! `Shared::state` -> compute queue cell (one at a time) -> port locks -> availability domains; +//! `Shared::state` -> UAT shared state -> VM exec lock -> GEM reservation. `RoleState::ack` +//! nests inside nothing. Capacity waits block on an availability domain signalled by the +//! release that provides the capacity, never on a sleep. mod availability; mod buffer; @@ -831,6 +835,12 @@ struct Shared { #[pin] feed_work: Work, preparations: Arc, + /// Compute admission waiting for a physical channel: signalled when a channel becomes + /// reusable or shareable, a QID or slot is returned, or a pool vacancy is witnessed. + channels: Arc, + /// Compute admission waiting for retirement-bound capacity (work states, ring and + /// in-flight room, drain witnesses, pool releases) or for publication to reopen. + progress: Arc, /// Logical-queue creation waiting for a returned queue slot or execution context. queues: Arc, /// Unbound compute publishers which can reuse a retired channel. @@ -1115,8 +1125,10 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = Self::max_frequency_khz(dev)?; - let preparations = preparation::Gate::new()?; + let channels = availability::Availability::new()?; + let progress = availability::Availability::new()?; let queues = availability::Availability::new()?; + let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); @@ -1147,6 +1159,8 @@ impl Gpu { feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + channels, + progress, queues, compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index e014d4bbb6542..0df2c5ae2ebb2 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -60,6 +60,7 @@ pub(super) struct Gate { } impl Gate { + #[cfg(CONFIG_KUNIT)] pub(super) fn new() -> Result> { Self::with_waiters(None) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5e6f20a3da43a..da0123d8405df 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -550,6 +550,8 @@ pub(crate) struct Queue { config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, + /// Prepared packets of the bound owner about to publish (see [`Self::claim`]). + claims: u32, last_published: u64, last_end: u64, released: bool, @@ -615,6 +617,7 @@ impl Queue { config_cache: None, ordinal: 0, submitted: 0, + claims: 0, last_published: 0, last_end: 0, released: false, @@ -2367,14 +2370,33 @@ impl Queue { Ok(self.reusable()) } + /// Another owner could take this channel now: idle and released, shareable, or vacant. + pub(crate) fn channel_free(&self) -> bool { + self.reusable() || self.shareable() || self.can_rearm() + } + + /// A pool replacement is in flight for this queue. + pub(crate) fn growing(&self) -> bool { + self.growing + } + /// The queue's first command has been published: its identity and configuration exist. pub(crate) fn installed(&self) -> bool { self.installed } + /// A packet of the bound owner passed prepare_job and will publish here. + pub(crate) fn claim(&mut self) { + self.claims = self.claims.saturating_add(1); + } + + pub(crate) fn release_claim(&mut self) { + self.claims = self.claims.saturating_sub(1); + } + /// An open client's idle channel that [`Self::share_idle`] can hand over. pub(crate) fn shareable(&self) -> bool { - self.installed && self.owner.is_some() && self.retirement_ready + self.claims == 0 && self.installed && self.owner.is_some() && self.retirement_ready && !self.growing && !self.quarantined && !self.awaiting_witness && self.exit.is_none() && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() && self.pool_phase == PoolPhase::Populated diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 88a3330155cac..f1cefb99fe005 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -299,6 +299,38 @@ impl Registry { Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) } + /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks + /// only whether some channel is free for a new owner. + fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { + if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { + return if queue.spared_pending() { + Readiness::Progress + } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { + // Publication reports the mismatch. + Readiness::Ready + } else if queue.pool_fits(required) || !queue.growing() { + Readiness::Ready + } else { + Readiness::Progress + }; + } + let teardown = &self.teardown; + let available = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + (queue.reusable() || queue.shareable() || queue.can_rearm()) + && !queue.context().is_some_and(|bound| teardown.releasing(bound)) + }); + if available || self.qids.compute_fits(self.admission.queues() as usize) { + Readiness::Ready + } else { + Readiness::Channel + } + } + /// Binds `owner` to a reusable channel whose last owner released it, or /// names a vacant one to re-arm. None if no channel is reusable. fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { @@ -563,6 +595,16 @@ impl Registry { } } +/// What a compute packet still lacks before publication. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(super) enum Readiness { + Ready, + /// A physical channel: every one is busy and no QID fits. + Channel, + /// Retirement-bound capacity or reopened publication. + Progress, +} + /// Outcome of offering an owner a retained compute queue. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Rebind { @@ -616,6 +658,10 @@ pub(super) struct Backend { /// binding under the cell lock; a stale cell only sends the packet back to that path. #[pin] direct: kernel::sync::Mutex>>, + /// The channel a prepared packet is about to publish into. Its claim keeps the idle + /// channel from being shared away between prepare_job and run_job. + #[pin] + claimed: kernel::sync::Mutex>>, } impl Backend { @@ -707,6 +753,7 @@ impl Backend { compute_backlog: core::sync::atomic::AtomicU32::new(0), closed: core::sync::atomic::AtomicBool::new(false), direct <- kernel::new_mutex!(None, "g17::Backend::direct"), + claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), }), GFP_KERNEL, )?, @@ -852,11 +899,14 @@ impl Backend { drop(queue); let mut state = self.shared.state.lock(); let firmware = state.as_deref_mut().ok_or(ENODEV)?; - firmware.queues.cancel_compute(reservation) + firmware.queues.cancel_compute(reservation)?; + drop(state); + // The slot and its never-published QID are free again. + self.shared.channels.notify(); + Ok(()) } fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { - use kernel::time::{delay::fsleep, Delta}; admission.check()?; { let state = self.shared.state.lock(); @@ -874,6 +924,9 @@ impl Backend { let mut deferred = DeferredBatch::publication()?; let mut growth = None; let result = (|| loop { + // Growth waits for this queue's retirement and for firmware to consume the pool + // release; both are progress the event worker reports. + let observed = self.shared.progress.observe(); let result = (|| { let mut state = self.shared.state.lock(); admission.check()?; @@ -899,9 +952,9 @@ impl Backend { match result { Ok(old) => { drop(old); return Ok(()); } Err(error) if error == EAGAIN && lease.is_current() => { - self.shared.queue_events(); - fsleep(Delta::from_millis(1)); - } + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; + } Err(error) => return Err(error), } })(); @@ -948,6 +1001,8 @@ impl Backend { } loop { admission.check()?; + // Sampled before any capacity check of this attempt. + let observed = self.shared.channels.observe(); let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; let (uat, objects, reservation) = { @@ -966,7 +1021,7 @@ impl Backend { Err(error) if error == ENOSPC => { drop(state); drop(lease); - self.wait_compute_capacity(admission, true)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -984,7 +1039,7 @@ impl Backend { // it cannot create a QID and forces fresh allocation. drop(state); drop(lease); - self.wait_compute_capacity(admission, false)?; + self.wait_channels(observed, admission, false)?; continue; } Err(error) if error == ENOSPC => { @@ -1018,7 +1073,7 @@ impl Backend { let (pages, ring) = match built { Err(error) if error == ENOSPC => { drop(lease); - self.wait_compute_capacity(admission, true)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -1108,7 +1163,7 @@ impl Backend { } drop(lease); if result == Err(ENOSPC) { - self.wait_compute_capacity(admission, true)?; + self.wait_channels(observed, admission, true)?; continue; } if result != Err(EAGAIN) { return result; } @@ -1237,6 +1292,10 @@ impl DeferredBatch { } Ok(()) } + /// Completions collected so far. + pub(super) fn pending(&self) -> usize { + self.values.len() + } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { use super::completion::Deferred; @@ -1570,12 +1629,15 @@ impl super::Firmware { )?; } let port = &self.port; + let retired = deferred.pending(); + let mut freed = false; for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.latch_observations(&mut registry.observations, masks)?; queue.retire( masks, @@ -1584,6 +1646,7 @@ impl super::Firmware { |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. @@ -1606,11 +1669,22 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.release_witnessed( state, |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); + } + // Wake only the admission waits this batch can end. The fences gate + // scheduler entities alone; their callbacks never take the device mutex. + let shared = &self.primary.state.shared; + if freed { + shared.channels.notify(); + } + if deferred.pending() != retired { + shared.progress.notify(); } Ok(()) } @@ -1620,11 +1694,10 @@ impl Backend { fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; - use kernel::time::{delay::fsleep, Delta}; - const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; - for _ in 0..SPARED_WAIT_MS { + loop { admission.check()?; + let observed = self.shared.progress.observe(); let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -1641,14 +1714,12 @@ impl Backend { result => return result, } } - // Other owners continue submitting while this owner gives its - // queue-local drain witness time to arrive. - fsleep(Delta::from_millis(1)); + // Other owners continue submitting while this owner waits for its + // queue-local drain witness. No new command has been accepted: the + // spared queue and its retained work stay intact past the deadline. + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; } - // No new command has been accepted. Keep the spared queue and its - // retained work intact until a later admission observes its witness. - admission.check()?; - Err(EBUSY) } /// Steady-state publication into this owner's bound channel without the device mutex: only @@ -1706,6 +1777,98 @@ impl Backend { } } + /// Scheduler admission of a compute packet. Shared capacity the packet lacks (a channel, a + /// work state, retirement-bound room, reopened publication) becomes a fence that the release + /// providing it signals, so publication finds it ready instead of sleeping for it. Errors + /// and anything prepare cannot judge are left to publication. + pub(super) fn prepare_compute(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()).ok()?; + loop { + let channels = self.shared.channels.observe(); + let progress = self.shared.progress.observe(); + let (domain, observed) = match self.compute_readiness(packet, required) { + Readiness::Ready => return None, + Readiness::Channel => (&self.shared.channels, channels), + Readiness::Progress => (&self.shared.progress, progress), + }; + self.shared.queue_events(); + match domain.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + + fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { + match packet.completion.work_state().and_then(|work| work.acquire()) { + Err(error) if error == EBUSY => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(_) => {} + } + // The steady state: this owner's channel is bound and installed. + if packet.completion.sources().is_none() { + if let Some(cell) = self.direct.lock().clone() { + let bound = cell.lock().is_some_and(|mut queue| { + let bound = queue.installed() + && !queue.spared_pending() + && queue.pool_fits(required) + && matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)); + if bound { + queue.claim(); + } + bound + }); + if bound { + self.hold_claim(Some(cell)); + return Readiness::Ready; + } + } + } + match self.sources_published(packet) { + Ok(false) => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(true) => {} + } + let lease = match self.shared.preparations.try_enter() { + Ok(lease) => lease, + Err(error) if error == EAGAIN => return Readiness::Progress, + Err(_) => return Readiness::Ready, + }; + let state = self.shared.state.lock(); + let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; + if !lease.is_current() + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 + { + return Readiness::Progress; + } + firmware.queues.compute_readiness(self.owner, Some(&self.context), required) + } + + /// Replaces the prepared packet's channel claim; None releases it after publication, + /// cancellation or close. + pub(super) fn hold_claim(&self, cell: Option>) { + let old = core::mem::replace(&mut *self.claimed.lock(), cell); + let freed = old.as_ref().and_then(|old| old.lock()).is_some_and(|mut queue| { + let busy = !queue.channel_free(); + queue.release_claim(); + busy && queue.channel_free() + }); + if freed { + // A channel nobody prepares into any more is shareable now. + self.shared.channels.notify(); + } + } + /// Called with the device mutex held after a successful publication: remembers the bound /// channel for direct publication. fn remember_direct(&self, registry: &Registry) { @@ -1868,10 +2031,11 @@ impl Backend { } // A bound queue needs none of ensure_compute's admission passes. Try // without them; the publication section below checks the same state. + // Every retry blocks on the release that ends its wait: retirement + // progress (work states, ring room, replays) and the reopening of + // publication after recovery both signal `progress`. let mut bound = true; let mut demand = None; - let mut recovery_polls = 0; - let mut state_delay = 20i64; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } @@ -1880,11 +2044,13 @@ impl Backend { Ok(()) }; let admission = Admission::new(deadline, &check); + let progress = &self.shared.progress; loop { admission.check()?; + let observed = progress.observe(); if !self.sources_published(packet)? { self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } // All DRM dependencies were satisfied before this scheduler callback. @@ -1897,15 +2063,11 @@ impl Backend { } let lease = self.shared.preparations.enter_until(&admission)?; // Work states free as this context's earlier commands retire. - // Wait for one under the packet's admission deadline, as render - // publication does, instead of failing the packet. match packet.completion.work_state()?.acquire() { Err(error) if error == EBUSY => { drop(lease); - admission.check()?; self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); + self.wait_release(progress, observed, &admission)?; continue; } result => { @@ -1928,14 +2090,10 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } firmware.replay_compute(deferred)?; @@ -1955,28 +2113,18 @@ impl Backend { dependencies.as_slice(), deferred, ); - if matches!(result, Err(EBUSY) | Err(EAGAIN)) - && !packet.is_published() - && (firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0) + if !packet.is_published() + && (matches!(result, Err(EAGAIN)) + || (matches!(result, Err(EBUSY)) + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0))) { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; - drop(state); - drop(lease); - self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; - } - if matches!(result, Err(EAGAIN)) && !packet.is_published() { - // Backpressure owns no firmware-visible attempt. Other queues - // can submit and the event worker can retire while we sleep. + // Recovery or backpressure: nothing is firmware-visible. Other + // queues keep submitting and the event worker retires meanwhile. drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } if result.is_ok() { @@ -2103,6 +2251,8 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { + let mut freed = false; + let mut releasing = false; for mut queue in self .queues .compute @@ -2110,13 +2260,32 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.recycle_killed()?; queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } + freed |= busy && queue.channel_free(); + releasing |= queue.pool_release_pending(); + } + freed |= self.queues.observe_pool_releases(&self.init, deferred)? != 0; + let shared = &self.primary.state.shared; + // Edge notifications cover the transitions this driver makes; every + // event pass also rechecks the level, so capacity that appeared through + // another engine (render QIDs, closed logical queues, render-held work + // states) still ends a scheduler wait that has no timeout of its own. + if freed + || (shared.channels.has_waiters() + && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) + { + shared.channels.notify(); + } + // A pool release has no completion event: each observation pass is + // the progress a growing owner waits for. + if releasing || shared.progress.has_waiters() { + shared.progress.notify(); } - self.queues.observe_pool_releases(&self.init, deferred)?; if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) .any(|queue| queue.pool_release_stalled(now)) { return Err(ETIMEDOUT); @@ -2182,6 +2351,47 @@ impl Backend { } fn wait_compute_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { + self.reclaim_capacity(admission, reclaim_backing)?; + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + admission.check() + } + + /// Compute admission without a channel: give back what pressure allows, then block until a + /// channel is released after `observed` (sampled before the failed check). + fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { + self.reclaim_capacity(admission, reclaim_backing)?; + self.wait_release(&self.shared.channels, observed, admission) + } + + /// Blocks until `domain` is signalled after `observed`, this packet's admission ends, or one + /// admission slice passes: cancellation and logical close have no release to wait for. + fn wait_release( + &self, + domain: &super::availability::Availability, + observed: u64, + admission: &Admission<'_>, + ) -> Result { + use kernel::dma_fence::RawDmaFence; + let slice = admission.slice_ms()?; + if let Some(fence) = domain.wait(observed)? { + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(slice) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + } + admission.check() + } + + /// Reclaim steps of a capacity wait that never sleep: observe consumed pool releases and, + /// under backing pressure, give idle pools back. + fn reclaim_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { admission.check()?; let mut deferred = DeferredBatch::publication()?; let result: Result = (|| { @@ -2216,8 +2426,6 @@ impl Backend { Err(error) if error == EAGAIN => {} result => result?, } - admission.check()?; - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); admission.check() } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index b393afd9c8df2..39f628d007e18 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -55,6 +55,7 @@ impl Backend { drop(state); drop(binding); self.finish_uninstalled_compute(retired)?; + self.shared.channels.notify(); self.shared.queue_events(); return Ok(()); } @@ -92,6 +93,7 @@ impl Backend { // A builder may own only an off-lock reservation when close arrives. // Record intent before scanning queues; every later install/rearm checks it. self.closed.store(true, Ordering::Release); + self.hold_claim(None); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -106,6 +108,8 @@ impl Backend { firmware.queues.release_render(self.owner)?; firmware.queues.admission.release_owner(self.owner); drop(state); + // Its idle channel, or its slot and never-published QID, is free. + self.shared.channels.notify(); self.shared.queue_events(); Ok(()) } @@ -212,8 +216,12 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - fn prepare(&self, _packet: &Packet) -> Option { - None + /// Compute admission waits for its shared capacity here, as a fence. + fn prepare(&self, packet: &Arc) -> Option { + match packet.engine() { + Engine::Compute => self.prepare_compute(packet), + Engine::Render => None, + } } fn publish(&self, packet: Arc) -> Result { @@ -241,6 +249,9 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -269,6 +280,9 @@ impl job::Backend for Backend { fn cancel(&self, packet: Arc) { self.release_backlog(&packet); + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 0a8ebc28c02e2..32b9acef94c0f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -104,6 +104,8 @@ impl Firmware { } } self.queues.replay_outer_wait = None; + // Publication behind these replays may proceed. + self.primary.state.shared.progress.notify(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index aaa2e281c69ee..16bd5828265a4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -190,10 +190,17 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); + let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? .is_some() - {} + { + consumed = true; + } + if consumed { + // A channel whose last scheduler release is consumed can change owner. + self.primary.state.shared.channels.notify(); + } Ok(()) } } From 327e50fc45b57acf1954e8c2d78977198286feb0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:52:55 -0400 Subject: [PATCH 20/64] drm/asahi: wait for a G17 compute source only through its completion record A consumer of a firmware-visible compute source carried a completion-wait record before its descriptor and also named the source in its kick entry. The second dependency flagged the command as entry-signalled, so it also appended a predecessor record, and the firmware resolved the same wait twice. Keep only the completion-wait record, which orders the consumer behind its source by itself. Queue-to-queue hop 49.6 -> 46.0 us. Cross-queue order and data checks (10k iterations each), dependent reads under pressure, the compute/sync CTS gate and mixed churn pass. (cherry picked from commit 58ef65a530b707fb77878e9fdc68d577a27759fc) --- drivers/gpu/drm/asahi/g17/runtime.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index f1cefb99fe005..2c0d91820c979 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1461,7 +1461,9 @@ impl super::Firmware { let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { return Err(EAGAIN); }; - merged.insert(sequence, dependency)?; + // The completion-wait record orders the consumer behind its + // source; naming the source in the kick entry as well only adds + // a second, slower wait and a predecessor record. let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; if let Some(old) = native.iter_mut().find(|old| old.dependency.qid() == dependency.qid()) { From 6f4d3bbb47c1a7b19abef118b41b131440a8f414 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 23:11:57 -0400 Subject: [PATCH 21/64] drm/asahi: add G17 command lifecycle tracepoints Add tracepoints for each command's publication (doorbell rung, with its out fence and the CPU counter), its retirement (with the firmware's GPU start and end timestamps), physical queue binds and event worker pass boundaries. With the counter and the GPU's system counter timestamps on the same events, a trace places firmware dispatch, GPU execution, firmware completion and every host stage of a command on one timeline. The events are disabled by default and cost a static-key test each. (cherry picked from commit 565dae906223948df97a252cc3274e8db6183363) --- drivers/gpu/drm/asahi/Makefile | 3 +- drivers/gpu/drm/asahi/g17/mod.rs | 1 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 8 ++ drivers/gpu/drm/asahi/g17/queue/render.rs | 4 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 3 + drivers/gpu/drm/asahi/g17/trace.c | 38 +++++++ drivers/gpu/drm/asahi/g17/trace.h | 114 ++++++++++++++++++++ drivers/gpu/drm/asahi/g17/trace.rs | 56 ++++++++++ 8 files changed, 226 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/asahi/g17/trace.c create mode 100644 drivers/gpu/drm/asahi/g17/trace.h create mode 100644 drivers/gpu/drm/asahi/g17/trace.rs diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 8f8f679532d2b..9edbc6ba3b8d4 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o g17/sync_input.o +asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/trace.o +CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G16) += g16/glue.o diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 330f57ab59779..3d7f58ea08ecf 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -31,6 +31,7 @@ pub(crate) mod context; mod data_segment; mod dependency; mod sync_input; +mod trace; mod event; mod feed; mod fence; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index da0123d8405df..801e69c97ef79 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -589,6 +589,7 @@ impl Queue { let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, kick.id().qid(), context.id(), 0); Ok(Self { owner: Some(owner), retained_owner: None, @@ -924,6 +925,10 @@ impl Queue { self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; + if let Some(ticket) = front.ticket { + crate::g17::trace::complete(crate::g17::trace::COMPUTE, ticket.qid, + ticket.ordinal, &front.packet, [timestamps[0], timestamps[1]]); + } self.retire_pending = true; defer(Deferred::Retired( front.packet.completion.clone(), @@ -1316,6 +1321,8 @@ impl Queue { } context.mark_published(); context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, self.qid(), context.id(), + if same_vm { 2 } else { 1 }); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -2151,6 +2158,7 @@ impl Queue { } fence(Ordering::SeqCst); host.notify((0x83 << 48) | 0x10)?; + crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); if let Err(error) = result { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index e063ef3a047fc..faf31514a2f2f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -954,6 +954,8 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; + crate::g17::trace::publish(crate::g17::trace::RENDER, self.qids()[1], self.ordinal as u32, + packet); host.note_submission() })(); if let Err(error) = publish { @@ -1236,6 +1238,8 @@ impl Pair { front.qos_completed = true; front.flist_completed = true; } + crate::g17::trace::complete(crate::g17::trace::RENDER, qids[1], ordinal as u32, + &front.packet, [timestamps[0], timestamps[3]]); defer(Deferred::Retired( front.packet.completion.clone(), Ok(timestamps), diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index e46c820efa155..b1e8f82d858f9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -323,8 +323,10 @@ impl crate::g17::Shared { }; deferred }; + crate::g17::trace::event(0); let result = self.runtime_event_turn(&mut deferred); deferred.finish(); + crate::g17::trace::event(2); self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); @@ -352,6 +354,7 @@ impl crate::g17::Shared { (recover, masks) }; deferred.finish(); + crate::g17::trace::event(1); if recover { self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c new file mode 100644 index 0000000000000..feffb96315845 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#define CREATE_TRACE_POINTS +#include "trace.h" + +bool asahi_g17_trace_publish_enabled(void) +{ + return trace_asahi_g17_publish_enabled(); +} + +void asahi_g17_trace_publish(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + struct dma_fence *fence) +{ + trace_asahi_g17_publish(engine, qid, ordinal, ctx, seq, + fence ? fence->context : 0, fence ? fence->seqno : 0, + __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_complete(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end) +{ + if (trace_asahi_g17_complete_enabled()) + trace_asahi_g17_complete(engine, qid, ordinal, ctx, seq, gpu_start, + gpu_end, __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind) +{ + trace_asahi_g17_bind(engine, qid, ctx, kind); +} + +void asahi_g17_trace_event(u8 phase) +{ + if (trace_asahi_g17_event_enabled()) + trace_asahi_g17_event(phase, __arch_counter_get_cntvct()); +} diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h new file mode 100644 index 0000000000000..e1ef55715e312 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Command lifecycle tracepoints of the G17 runtime. Times are joined by + * (ctx, seq); `counter` is the CPU's virtual counter and `gpu_start`/`gpu_end` + * come from the firmware's completion record, so a reader can place firmware + * dispatch and completion on the host timeline. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM asahi_g17 + +#if !defined(ASAHI_G17_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define ASAHI_G17_TRACE_H + +#include + +TRACE_EVENT(asahi_g17_publish, + TP_PROTO(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 fence_context, u64 fence_seqno, u64 counter), + TP_ARGS(engine, qid, ordinal, ctx, seq, fence_context, fence_seqno, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ordinal) + __field(u32, ctx) + __field(u64, seq) + __field(u64, fence_context) + __field(u64, fence_seqno) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ordinal = ordinal; + __entry->ctx = ctx; + __entry->seq = seq; + __entry->fence_context = fence_context; + __entry->fence_seqno = fence_seqno; + __entry->counter = counter; + ), + TP_printk("engine=%u qid=%u ordinal=%u ctx=%u seq=%llu fence=%llu:%llu counter=%llu", + __entry->engine, __entry->qid, __entry->ordinal, __entry->ctx, + __entry->seq, __entry->fence_context, __entry->fence_seqno, + __entry->counter) +); + +TRACE_EVENT(asahi_g17_complete, + TP_PROTO(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end, u64 counter), + TP_ARGS(engine, qid, ordinal, ctx, seq, gpu_start, gpu_end, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ordinal) + __field(u32, ctx) + __field(u64, seq) + __field(u64, gpu_start) + __field(u64, gpu_end) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ordinal = ordinal; + __entry->ctx = ctx; + __entry->seq = seq; + __entry->gpu_start = gpu_start; + __entry->gpu_end = gpu_end; + __entry->counter = counter; + ), + TP_printk("engine=%u qid=%u ordinal=%u ctx=%u seq=%llu gpu=%llu..%llu counter=%llu", + __entry->engine, __entry->qid, __entry->ordinal, __entry->ctx, + __entry->seq, __entry->gpu_start, __entry->gpu_end, __entry->counter) +); + +TRACE_EVENT(asahi_g17_bind, + TP_PROTO(u8 engine, u8 qid, u32 ctx, u8 kind), + TP_ARGS(engine, qid, ctx, kind), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ctx) + __field(u8, kind) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ctx = ctx; + __entry->kind = kind; + ), + TP_printk("engine=%u qid=%u ctx=%u kind=%u", + __entry->engine, __entry->qid, __entry->ctx, __entry->kind) +); + +TRACE_EVENT(asahi_g17_event, + TP_PROTO(u8 phase, u64 counter), + TP_ARGS(phase, counter), + TP_STRUCT__entry( + __field(u8, phase) + __field(u64, counter) + ), + TP_fast_assign( + __entry->phase = phase; + __entry->counter = counter; + ), + TP_printk("phase=%u counter=%llu", __entry->phase, __entry->counter) +); + +#endif + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace +#include diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs new file mode 100644 index 0000000000000..a616b68735225 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Command lifecycle tracepoints (tracefs `events/asahi_g17`). Each call is a +//! static-key test in C while its event is disabled. + +use kernel::{bindings, dma_fence::RawDmaFence, prelude::*, sync::Arc}; + +unsafe extern "C" { + fn asahi_g17_trace_publish_enabled() -> bool; + fn asahi_g17_trace_publish(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, + fence: *mut bindings::dma_fence); + fn asahi_g17_trace_complete(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, + gpu_start: u64, gpu_end: u64); + fn asahi_g17_trace_bind(engine: u8, qid: u8, ctx: u32, kind: u8); + fn asahi_g17_trace_event(phase: u8); +} + +pub(super) const RENDER: u8 = 0; +pub(super) const COMPUTE: u8 = 1; + +/// Called once the command's doorbell has been rung. +pub(super) fn publish(engine: u8, qid: u8, ordinal: u32, packet: &Arc) { + // SAFETY: Pure static-key query. + if !unsafe { asahi_g17_trace_publish_enabled() } { + return; + } + let fence = packet.completion.fence(); + // SAFETY: `fence` holds a reference for the duration of the call. + unsafe { + asahi_g17_trace_publish(engine, qid, ordinal, packet.context.id(), packet.order.sequence, + fence.raw()) + }; +} + +/// Called when exact completion with GPU timestamps is retired. +pub(super) fn complete(engine: u8, qid: u8, ordinal: u32, packet: &super::job::Packet, + gpu: [u64; 2]) +{ + // SAFETY: Plain values only. + unsafe { + asahi_g17_trace_complete(engine, qid, ordinal, packet.context.id(), packet.order.sequence, + gpu[0], gpu[1]) + }; +} + +/// A physical queue was bound to a logical owner: 0 fresh, 1 rebound, 2 same VM. +pub(super) fn bind(engine: u8, qid: u8, ctx: u32, kind: u8) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_bind(engine, qid, ctx, kind) }; +} + +/// Event worker pass boundaries: 0 begin, 1 fences signalled, 2 end. +pub(super) fn event(phase: u8) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_event(phase) }; +} From e24e3bad9e7d3c6cf7079d683c9364d619cd3a55 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 23:07:01 -0400 Subject: [PATCH 22/64] drm/asahi: ring the G17 compute pipe doorbell for every publication Compute rang its pipe and priority doorbell (pipe 2 in bits 1:0, class in bits 3:2) only for the first publication of a priority class, then the generic work doorbell for every command. Render rings its own pipe doorbell for every publication, and so does the M5 compute path. Ring the compute pipe doorbell for every publication instead of the generic one. gpubench on Neo (us/submit, on the integration branch): compute.light d2 60.3 -> 53.2, d64 22.8 -> 21.5, d1 163 -> 154..160, procs d16 unchanged (27.2), hop unchanged (48.7). Stress gate passes. (cherry picked from commit 398141cf95c44ac4d0073f3620a99d1235f31309) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 801e69c97ef79..4189f687b0694 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -2120,7 +2120,6 @@ impl Queue { self.outer_started & ring == 0, )?, )?; - let notify_activation = self.outer_started & ring == 0; self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); @@ -2151,13 +2150,12 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - if notify_activation { - fence(Ordering::SeqCst); - // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; - } + // Every publication rings its own pipe and priority class, as render + // publication does: pipe 2 (compute) in bits 1:0, priority class in + // bits 3:2. The generic work doorbell left the firmware to find the + // ring itself and delayed dispatch. fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x10)?; + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); From 94c66ecf24e6d931a434914d6a513ee36ff6ffd7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 23:38:41 -0400 Subject: [PATCH 23/64] drm/asahi: map the G17 compute queue graph cached for the firmware Render keeps its queue records in a firmware-cached prefix; the compute queue record, job-list head and item ring were mapped uncached. Allocate the compute graph from the retained object pool with its 32 KiB queue record, job-list head and item ring in a firmware-cached prefix, and move the producer/consumer words to their own uncached page after it, as native does. compute.light d64 22.2 -> 19.7 us/submit, 16-process compute 27.2 -> 25.7, queue-to-queue hop 47.5 -> 42.4 us. Compute/sync CTS gate, cross-queue order/data 10k, dependent reads under pressure and mixed churn pass. (cherry picked from commit 488708add18c39f709d6a9b53736d339428cf5d0) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 39 +++++++++++++++------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 4189f687b0694..6ca63595154de 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -34,6 +34,10 @@ const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; const POINTERS: usize = 0x100; +/// Producer/consumer words live on their own uncached page after the graph, +/// which the firmware reads through a cached mapping (queue record, job-list +/// head and item ring), as native does. +const CURSORS: usize = GRAPH_SIZE; const ITEMS: usize = 0x4000; const STAMP: usize = 0x300; @@ -120,7 +124,7 @@ struct Graph { usage: KernelObject, operand: KernelObject, support: KernelObject, - graph: KernelObject, + graph: PooledObject, context_low: KernelObject, context_high: KernelObject, /// Announcements, which stay uncached for the firmware. @@ -157,11 +161,18 @@ impl Graph { support.write(0x100, support_va + page + 4)?; support.write(0x110, 0x50u32)?; support.write(mmu::UAT_PGSZ + 4, 1u32)?; - let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; + let mut graph = objects.allocate( + alloc, + Family::QueueState, + GRAPH_SIZE + mmu::UAT_PGSZ, + GRAPH_SIZE, + prot, + cpu, + )?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + POINTERS as u64, + pointers_va: graph_va + CURSORS as u64, items_va: graph_va + ITEMS as u64, job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), @@ -172,6 +183,10 @@ impl Graph { view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; + graph.initialize::(CURSORS, |view| { + *view = PointerBlock::new(&pointers)?; + Ok(()) + })?; let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; @@ -259,9 +274,9 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), - consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), - count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), + producer: self.graph.word(CURSORS + self.pointers.producer)?.load(Ordering::Relaxed), + consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), + count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), }) } fn stamp(&self) -> Result { @@ -315,7 +330,7 @@ impl Graph { } fence(Ordering::SeqCst); self.graph - .word(POINTERS + self.pointers.producer)? + .word(CURSORS + self.pointers.producer)? .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); @@ -1689,11 +1704,11 @@ impl Queue { let producer = self .graph .graph - .word(POINTERS + self.graph.pointers.producer)? + .word(CURSORS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + target)? + .word(CURSORS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1764,7 +1779,7 @@ impl Graph { }; (object, slot * RECORD_STRIDE + offset) } else { - (&mut self.graph, 0x6800 + offset) + (&mut *self.graph, 0x6800 + offset) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -1790,7 +1805,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -2262,7 +2277,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); From db365ecc299f0663db8a629e70dfdb4d117eb95e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 23:28:59 -0400 Subject: [PATCH 24/64] drm/asahi: wait for G17 render capacity on the release that frees it Render admission and publication still slept or polled: a 20 us..1 ms backoff when every work state of the context was taken, up to 200 x 1 ms while recovery was observed, an unconditional 1 ms sleep for render capacity (through the compute capacity wait), and a 1 ms condvar timeout as the wakeup for a pair on loan, firmware room or a pending teardown. Block on the event that ends each wait instead: - a render availability domain, signalled by every admission-pool return (lease drop, owner close, quarantine change, exit-kill hold release, processor stop), every pair return from a loan, and on each event pass while a render publisher waits, since firmware ring room, consumed releases and teardown steps have no event of their own; - a per-context work-state domain, signalled whenever a work state returns; - the preparation gate: a publisher that observed recovery waits for its closure (close() now wakes the gate's waiters) and then blocks in enter() until recovery reopens it. Render capacity first returns idle backing, then waits on the domain. Waiters sample a domain only after a failed check and retry once before blocking, so steady-state publication takes no extra lock. Deadlines only bound the waits; 10 ms slices only bound cancellation checks. Steady state is unchanged (render d1 135.7, d32 41.4, queues d8 62.6 us/submit); the mild stress gate passes. (cherry picked from commit a3fc054bc52c2bc82350e6073366045a1c3c9489) --- drivers/gpu/drm/asahi/g17/context.rs | 27 ++- drivers/gpu/drm/asahi/g17/mod.rs | 18 +- drivers/gpu/drm/asahi/g17/preparation.rs | 31 +++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 24 ++- drivers/gpu/drm/asahi/g17/runtime.rs | 16 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 + .../drm/asahi/g17/runtime/render/submit.rs | 192 +++++++++++++----- 7 files changed, 236 insertions(+), 77 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 564d761d37f97..0d3948a2b69ff 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -122,6 +122,8 @@ pub(crate) struct Context { qos: QosLease, owner_pid: u32, states: WorkStates, + /// Signalled when a work state returns; publishers that found none wait on it. + state_returns: Arc, work: Arc>, work_base: u64, render: Arc>>>, @@ -140,6 +142,7 @@ impl Context { execution: Arc, owner_pid: u32, policy: Policy, + state_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -199,6 +202,7 @@ impl Context { qos, owner_pid, states: WorkStates::new(), + state_returns, work, work_base, render, @@ -378,8 +382,9 @@ impl WorkStateLease { Err(EBUSY) }; }; - if let Err(error) = self.context.work.lock().initialize(candidate) { - self.context.states.release(candidate); + let initialized = self.context.work.lock().initialize(candidate); + if let Err(error) = initialized { + self.context.release_state(candidate); return Err(error); } match self @@ -388,11 +393,15 @@ impl WorkStateLease { { Ok(_) => Ok(candidate), Err(installed) => { - self.context.states.release(candidate); + self.context.release_state(candidate); Ok(installed) } } } + /// The domain a publisher samples before acquire() and waits on after EBUSY. + pub(crate) fn returns(&self) -> &Arc { + &self.context.state_returns + } pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) } @@ -433,7 +442,17 @@ impl Drop for WorkStateLease { fn drop(&mut self) { let word = self.word(); if word != FREE { - self.context.states.release(word); + self.context.release_state(word); + } + } +} + +impl Context { + /// Every return bumps the domain: a publisher that sampled it before its + /// failed acquire() then never sleeps through this node. + fn release_state(&self, word: u32) { + if self.states.release(word) { + self.state_returns.notify(); } } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 3d7f58ea08ecf..5e4c0dcc756cb 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -62,7 +62,6 @@ use core::ops::Range; use core::sync::atomic::{ AtomicBool, AtomicU32, - AtomicU64, Ordering, // }; @@ -844,6 +843,11 @@ struct Shared { progress: Arc, /// Logical-queue creation waiting for a returned queue slot or execution context. queues: Arc, + /// Render publication waiting for render capacity, a pair on loan, a work + /// state or firmware room: signalled when a pair, lease or work state + /// returns, an owner closes, a teardown parks, and on every event pass + /// while a render publisher waits (firmware consumption has no event). + renders: Arc, /// Unbound compute publishers which can reuse a retired channel. compute_demand: AtomicU32, #[pin] @@ -868,9 +872,6 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, - /// Counts `changed` notifications, so that a waiter which checked under - /// the device mutex can tell whether it already missed one. - changes: AtomicU64, /// The port of the session in `state`; null before boot and once detachment begins. port: core::sync::atomic::AtomicPtr, } @@ -964,13 +965,10 @@ impl Shared { /// Wakes the waiters for a returned pair, binding or capacity change. fn note_changed(&self) { - self.changes.fetch_add(1, Ordering::AcqRel); self.changed.notify_all(); + self.renders.notify(); } - fn changes(&self) -> u64 { - self.changes.load(Ordering::Acquire) - } /// The session's port, valid while `lease` is held: shutdown and device loss join every /// lease before they detach the session. @@ -1130,6 +1128,7 @@ impl Gpu { let progress = availability::Availability::new()?; let queues = availability::Availability::new()?; let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; + let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); @@ -1163,6 +1162,7 @@ impl Gpu { channels, progress, queues, + renders, compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), @@ -1173,7 +1173,6 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), - changes: AtomicU64::new(0), port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, @@ -1325,6 +1324,7 @@ impl Gpu { shared.descriptor_flags, shared.variant, Some(shared.queues.clone()), + Some(shared.renders.clone()), ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 0df2c5ae2ebb2..3cafecf5d8206 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -179,8 +179,39 @@ impl Gate { } } + /// Also wakes publishers in wait_closure(); they then block in enter(). pub(super) fn close(&self) { self.invalidate(0); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + /// Opaque snapshot for wait_closure(), taken where recovery was observed. + pub(super) fn snapshot(&self) -> u64 { + self.state.load(Ordering::Acquire) + } + + /// A publisher that observed recovery (host pending, or the firmware's state + /// ahead of the event worker, which always follows with a restart and the + /// closure) sleeps until this gate has closed since `seen`. enter() then + /// blocks until recovery reopens it. The deadline is only a failure bound. + pub(super) fn wait_closure(&self, seen: u64, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + loop { + let state = self.state.load(Ordering::Acquire); + if state & (REMOVED | FAILED) != 0 { + return Err(ENODEV); + } + if state & CLOSED != 0 || state & !COUNT_MASK != seen & !COUNT_MASK { + return Ok(()); + } + let remaining = deadline.saturating_sub(super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + let ms = u32::try_from(remaining.div_ceil(1_000_000)).unwrap_or(u32::MAX); + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(ms)); + } } pub(super) fn remove(&self) { diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index ff2b294b6f35c..424bb1b7601f5 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -379,15 +379,19 @@ pub(crate) struct Pool { max_queues: u32, /// Logical-queue creation waiting for a returned slot. waiters: Option>, + /// Render publication waiting for render capacity: woken with every + /// release event above. + renders: Option>, } impl Pool { #[cfg(CONFIG_KUNIT)] pub(crate) fn new(max_queues: u32) -> Result> { - Self::with_waiters(max_queues, None) + Self::with_waiters(max_queues, None, None) } pub(crate) fn with_waiters( max_queues: u32, waiters: Option>, + renders: Option>, ) -> Result> { if max_queues > MAX_EXECUTION_CONTEXTS { return Err(EINVAL); @@ -399,10 +403,18 @@ impl Pool { queues: AtomicU32::new(0), max_queues, waiters, + renders, }), GFP_KERNEL, ) } + /// Every pair or lease return, owner close and quarantine change. + fn returned(&self) { + self.released.notify_all(); + if let Some(renders) = &self.renders { + renders.notify(); + } + } /// Logical queues currently counted, including one being created. pub(crate) fn queues(&self) -> u32 { self.queues.load(Ordering::Acquire) @@ -579,7 +591,7 @@ impl Pool { } } drop(state); - self.released.notify_all(); + self.returned(); } /// Active leases survive this transition. Their later drop is still the /// event which releases capacity, so a stopped owner is never double-freed. @@ -597,7 +609,7 @@ impl Pool { state.spared &= !released; state.kill_hold = 0; drop(state); - self.released.notify_all(); + self.returned(); } } @@ -715,7 +727,7 @@ impl RenderLease { } state.quarantined |= bit; drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. @@ -800,7 +812,7 @@ impl Drop for RenderLease { } state.release(self.slot); drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } } @@ -839,7 +851,7 @@ impl Pool { return; } self.state.lock().kill_hold &= !slots; - self.released.notify_all(); + self.returned(); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 2c0d91820c979..1bd500bae6cb0 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -192,16 +192,18 @@ impl Registry { descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, ) -> impl Init { - Self::with_waiters(clusters, deflake, descriptor_flags, variant, None) + Self::with_waiters(clusters, deflake, descriptor_flags, variant, None, None) } - /// `queues` wakes logical-queue creation when a logical queue slot is returned. + /// `queues` wakes logical-queue creation when a logical queue slot is returned; + /// `renders` wakes render publication when a render pair or lease returns. pub(super) fn with_waiters( clusters: u32, deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, queues: Option>, + renders: Option>, ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { @@ -210,7 +212,7 @@ impl Registry { buffers <- BufferIds::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues)?, + admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { @@ -735,7 +737,7 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, + Context::new(&alloc, &pool, &qos, execution, owner_pid, policy, shared.renders.clone())?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -2346,12 +2348,6 @@ impl Registry { } impl Backend { - pub(super) fn wait_capacity(&self, started: u64) -> Result { - let check = || self.admission_open(); - let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); - self.wait_compute_capacity(&admission, true) - } - fn wait_compute_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { self.reclaim_capacity(admission, reclaim_backing)?; kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index b1e8f82d858f9..1797ca5c0ce94 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -467,6 +467,11 @@ impl crate::g17::Shared { if grow_ready { self.queue_grow(); } + // Firmware ring room, consumed releases and teardown steps have no + // event of their own: recheck render publishers' conditions every pass. + if self.renders.has_waiters() { + self.renders.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 663e62e8f86fc..bd83c81b1d593 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -8,11 +8,7 @@ use crate::g17::{ queue::admission::{RenderLease, Request}, }; use core::sync::atomic::Ordering; -use kernel::{ - prelude::*, - sync::Arc, - time::{delay::fsleep, Delta}, -}; +use kernel::{prelude::*, sync::Arc}; /// Internal result of a publication attempted from the submitting ioctl that /// met contention before anything became visible: the scheduler publishes the @@ -60,6 +56,9 @@ impl Backend { }; let started = super::super::now_ns(); let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + // A capacity refusal samples the render domain and retries once; only + // a refusal after that sample waits, so no release can be missed. + let mut observed = None; loop { self.admission_open()?; if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } @@ -97,7 +96,7 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), false)? }; if reclaimed.is_none() { - self.wait_render_capacity(started, sleep)?; + self.wait_render_capacity(&mut observed, started, sleep)?; } drop(reclaimed); continue; @@ -131,7 +130,7 @@ impl Backend { let ready = firmware.queues.has_render_ids(self.owner); drop(state); if reclaimed.is_none() && !ready { - self.wait_render_capacity(started, sleep)?; + self.wait_render_capacity(&mut observed, started, sleep)?; } drop(reclaimed); } @@ -140,36 +139,129 @@ impl Backend { } } - /// A pair is borrowed, still settling, or waiting for the release of the - /// scheduler it last ran for to be consumed and the pair parked. The event - /// worker observes both, so kick it on every poll; time out only after the - /// same bound as a capacity wait. - fn wait_render_busy(&self, since: &mut Option, seen: u64, sleep: bool) -> Result { + /// Conditions which end a publication that has not become visible yet. + fn publication_open(&self, packet: &Packet) -> Result { + self.admission_open()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + Ok(()) + } + + /// Blocks on `domain` until a notification after `observed`, or the + /// deadline, which only bounds the wait. Slices of at most 10 ms let the + /// publisher observe cancellation, which has no notification here. + fn wait_domain( + &self, + domain: &crate::g17::availability::Availability, + observed: u64, + deadline_ns: u64, + packet: &Packet, + ) -> Result { + self.publication_open(packet)?; + let remaining = deadline_ns.saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + domain.block(observed, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// A pair is borrowed, still settling, waiting for firmware room or for the + /// release of the scheduler it last ran for. Every pair return, retirement + /// and teardown step signals the render domain, and each event pass does + /// while a publisher waits; the deadline is only the failure bound. + fn wait_render_busy( + &self, + since: &mut Option, + observed: &mut Option, + sleep: bool, + packet: &Packet, + ) -> Result { if !sleep { return Err(FALLBACK); } let now = super::super::now_ns(); - if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { - return Err(ETIMEDOUT); - } + let deadline = since.get_or_insert(now).saturating_add(super::super::WORK_RING_WAIT_NS); + // Sample, then retry once: only a refusal after the sample waits. + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; self.shared.queue_events(); - // Every pair return notifies under the device mutex: wake on it rather - // than sleeping out a fixed interval behind a short observation loan. - let mut state = self.shared.state.lock(); - if self.shared.changes() == seen { - let _ = self.shared.changed.wait_timeout(&mut state, kernel::time::msecs_to_jiffies(1)); - } - Ok(()) + self.wait_domain(&self.shared.renders, sample, deadline, packet) } - /// A closing owner may still hold its pair until exact retirement. Wait - /// outside both the device mutex and preparation gate so the event worker - /// and recovery can return capacity. No packet is firmware-visible here. - fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + /// Render capacity is short: return what idle owners hold, then block until + /// a pair, lease or owner returns. No packet is firmware-visible here, and + /// neither the device mutex nor a preparation lease is held while waiting. + fn wait_render_capacity(&self, observed: &mut Option, started: u64, sleep: bool) -> Result { if !sleep { return Err(FALLBACK); } - self.wait_capacity(started) + if self.reclaim_render_capacity()? { + *observed = None; + return Ok(()); + } + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; + self.shared.queue_events(); + self.admission_open()?; + let remaining = started + .saturating_add(super::super::WORK_RING_WAIT_NS) + .saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(EBUSY); + } + self.shared + .renders + .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// Gives idle render backing back under capacity pressure. Returns whether + /// anything was returned, in which case the caller retries at once. + fn reclaim_render_capacity(&self) -> Result { + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Ok(false); + } + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + let mut published = 0; + if !firmware.render_control_backpressured() { + (published, _) = firmware.queues.vacate_idle_pools( + &firmware.init, + 0, + super::super::now_ns(), + 0, + )?; + if published != 0 { + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + } + Ok(published != 0 + || !deferred.render.is_empty() + || !deferred.vacated.is_empty() + || !deferred.bindings.is_empty()) + })(); + deferred.finish(); + match result { + Err(error) if error == EAGAIN => Ok(false), + result => result, + } } fn render_with_lease( @@ -193,13 +285,12 @@ impl Backend { let admission = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - let mut state_polls = 0; - let mut state_delay = 20i64; let mut busy_since = None; - let mut recovery_polls = 0; let capacity_started = super::super::now_ns(); + let deadline_ns = capacity_started.saturating_add(super::super::WORK_RING_WAIT_NS); + let mut seen = None; + let mut states = None; loop { - let seen = self.shared.changes(); if packet.cancel_requested() { return Err(ECANCELED); } @@ -229,12 +320,15 @@ impl Backend { if !sleep { return Err(FALLBACK); } - if state_polls == 1000 { - return Err(ETIMEDOUT); + // Every node of this context belongs to unretired work: + // sample, retry once, then wait for the next returned one. + let work = packet.completion.work_state()?; + match states.take() { + None => states = Some(work.returns().observe()), + Some(sample) => { + self.wait_domain(work.returns(), sample, deadline_ns, packet)? + } } - state_polls += 1; - fsleep(Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); continue; } result => { @@ -243,40 +337,42 @@ impl Backend { } match self.ensure_render(packet, slot, sleep) { Err(error) if error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, capacity_started, sleep)?; continue; } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since, seen, sleep)?; + self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; continue; } result => result?, } match self.prepare_render(slot, packet, deferred, sleep) { Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, capacity_started, sleep)?; continue; } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { - let recovering = { + let (recovering, gate) = { let state = self.shared.state.lock(); let firmware = (*state).as_deref().ok_or(ENODEV)?; - firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + ( + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0, + self.shared.preparations.snapshot(), + ) }; if recovering { if !sleep { return Err(FALLBACK); } - if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { - return Err(ETIMEDOUT); - } - recovery_polls += 1; + // Recovery closes the preparation gate before touching + // any queue; the next pass blocks in enter() until it + // reopens. The worker raises the closure. self.shared.queue_events(); - fsleep(Delta::from_millis(1)); + self.shared.preparations.wait_closure(gate, deadline_ns)?; } else if error == EBUSY { - self.wait_render_busy(&mut busy_since, seen, sleep)?; + self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; } else if !sleep { return Err(FALLBACK); } From 06bc290c266abcb8b0d45f75c3369e231ae8c4a7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 23 Oct 2026 22:34:06 -0400 Subject: [PATCH 25/64] drm/asahi: give the G17 render scheduler the pair's depth as credits A logical render queue owns exactly one physical pair, whose active ring holds RENDER_DEPTH (16) commands, but its scheduler had 32 credits. Jobs 17 to 32 entered run_job and parked there in the admission pool (or in the 1 ms busy wait) until a retirement freed a ring entry, holding the scheduler worker. With the credit limit equal to the pair depth the scheduler itself waits for the credit that the retirement returns, and run_job never blocks on a full pair. render.light d32 41.15 -> 41.26 us/submit (unchanged; the row is firmware bound), all other rows unchanged, mild stress gate passes. (cherry picked from commit 15de925f91cdb2a875e09fc1c6b64af79fa37ae2) --- drivers/gpu/drm/asahi/g17/job.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index ae401b06f2eb3..572cfd5d0f382 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -14,7 +14,10 @@ use kernel::{ }; pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; -const RENDER_CREDITS: u32 = 32; +/// One logical render queue owns one physical pair, whose active ring holds +/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in +/// admission until a retirement; the scheduler waits for the credit instead. +const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; const COMPUTE_CREDITS: u32 = 16; #[derive(Clone, Copy, PartialEq, Eq)] From 3d1f2047766c2d6e8d165a4af2cfe4adc0edd366 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:03:31 -0400 Subject: [PATCH 26/64] drm/asahi: keep a G17 render free list registered while the firmware is busy A render pool whose last render retired sent its final FreeListRelease right away: a control-ring record and a control doorbell for every render of a one-deep queue. The firmware processes each such message serially before it returns to the work rings, so with several queues active the releases stretched every completion burst (one release per render in render.light queues: about 44.6k per run). When another pair still has published work, leave a registration whose references drop to zero lingering instead; the owner's next render reuses it like an overlapping render, with no release and no new generation. A lingering registration ends, and its release is published, when its owner closes or moves to another pool (each event pass checks) or after the 16 ms firmware idle hold, for which the event worker schedules a timed pass. On an otherwise idle device the release is still sent at once: reusing a registration that went idle costs the firmware more than the release does (render d1 147 -> 184 us when every idle registration lingered). render.light queues d2/d4/d8: 80.8/67.3/62.6 -> 76.9/62.4/58.2 us/submit; render d1 148, d32 41.3 (unchanged). Releases per queues-d8 run: 44574 -> 18 (measured with every idle registration lingering). Mild stress gate passes. --- drivers/gpu/drm/asahi/g17/freelist.rs | 37 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime/events.rs | 18 ++++++--- drivers/gpu/drm/asahi/g17/runtime/render.rs | 35 +++++++++++++++++- .../gpu/drm/asahi/g17/runtime/render/pools.rs | 34 +++++++++++++++-- 4 files changed, 111 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 3b31c3cead064..9015dc82e4076 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -670,10 +670,18 @@ pub(crate) struct Owner { enum LeasePhase { Preparing, Ready, + /// No reference since the given time. The registration stays live so that + /// the owner's next render reuses it without a control-ring release; the + /// release becomes pending when the linger ends. + Lingering(u64), ReleaseSent(u32), Released, } +/// How long an idle registration of an open owner may stay live: the +/// firmware's effort idle hold. +pub(crate) const LINGER_NS: u64 = 16_000_000; + /// References to one prepopulated owner: preparation and two engine commands /// per render. Published releases do not gate reacquisition; their consumption /// witnesses are carried into the successor. @@ -707,10 +715,14 @@ impl Lease { } let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; let same_owner = previous.owner == owner; + // A lingering registration was never released: it continues like an + // overlap of the owner's previous render. let overlapping = same_owner - && previous.phase == LeasePhase::Ready - && previous.references >= 2 - && previous.commands >= 2; + && match previous.phase { + LeasePhase::Ready => previous.references >= 2 && previous.commands >= 2, + LeasePhase::Lingering(_) => true, + _ => false, + }; let reacquire = previous.reacquirable(); if !overlapping && !reacquire { return Err(EBUSY); @@ -772,16 +784,35 @@ impl Lease { Err(EIO) } /// Both engines have retired this render's exact retained submission. + /// With `linger`, a registration left without references stays live + /// instead of making its final release pending. pub(crate) fn complete_render( &mut self, + linger: bool, ) -> Result> { if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { return Err(EINVAL); } self.references -= 2; self.commands -= 2; + if self.references == 0 && linger { + self.phase = LeasePhase::Lingering(super::now_ns().max(1)); + } Ok(self.pending_release()) } + /// The time since which an idle registration lingers, if it does. + pub(crate) fn lingering(&self) -> Option { + match self.phase { + LeasePhase::Lingering(since) => Some(since), + _ => None, + } + } + /// Ends a linger; the final release becomes pending. + pub(crate) fn end_linger(&mut self) { + if matches!(self.phase, LeasePhase::Lingering(_)) { + self.phase = LeasePhase::Ready; + } + } pub(crate) fn pending_release(&self) -> Option { (self.phase == LeasePhase::Ready && self.references == 0).then(|| { super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 1797ca5c0ce94..cdd8cc6d5dea5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -399,6 +399,11 @@ impl crate::g17::Shared { .any(|queue| queue.in_flight()); let in_flight = compute || firmware.queues.render_in_flight(); let idle = firmware.idle_check(now, in_flight)?; + let lingers = if firmware.recovery.pending() { + None + } else { + firmware.expire_render_lingers(now)? + }; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; let pools = firmware.service_compute_pools(deferred, now)?; @@ -441,11 +446,14 @@ impl crate::g17::Shared { } else { idle }; - // A retained compute pool gives its backing back when it expires. - let idle = match (idle, pools) { - (Some(delay), Some(expiry)) => Some(delay.min(expiry)), - (delay, expiry) => delay.or(expiry), - }; + // A retained compute pool gives its backing back when it expires, and + // so does an idle render free-list registration. + let idle = [pools, lingers] + .into_iter() + .fold(idle, |idle, expiry| match (idle, expiry) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }); if let Some(delay) = idle { self.queue_idle(delay); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 2877dea91d0ea..d770c433528c9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -609,7 +609,16 @@ impl crate::g17::queue::render::Host for Host<'_> { buffer_id: u8, ) -> Result { let queues = &mut self.firmware.queues; - let next = queues.render.pools.completed(pool_id, generation)?; + // Each final release is a control record the firmware processes before + // it returns to the work rings. While other pairs keep it busy, a + // registration left idle lingers instead, and the owner's next render + // reuses it. On an otherwise idle device the release is cheaper than + // reusing a registration that went idle, so it is sent at once. + // The pair retiring here is lent out of its entry meanwhile. + let linger = queues.render.entries.iter().flatten().any(|entry| { + entry.pair.as_ref().is_some_and(|pair| pair.published_in_flight()) + }); + let next = queues.render.pools.completed(pool_id, generation, linger)?; queues.buffers.can_release(buffer_id)?; if let Some(owners) = owners { self.firmware.port.qos_complete(&owners)?; @@ -645,6 +654,30 @@ impl crate::g17::Firmware { result } + /// Event-worker turn: ends free-list lingers that elapsed or whose owner + /// closed or moved on, publishes their releases, and returns the delay to + /// the next expiry, which the worker turns into a timed wake. + pub(in crate::g17) fn expire_render_lingers(&mut self, now: u64) -> Result> { + let render = &mut self.queues.render; + let entries = &render.entries; + let (ended, next) = render.pools.expire_lingers(now, |owner| { + entries.iter().flatten().any(|entry| { + !entry.closed + && entry + .pool + .as_ref() + .is_some_and(|pool| pools::Pools::owner(pool) == *owner) + }) + }); + if ended { + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + Ok(next) + } + fn release_render_lists(&mut self) -> Result { if self.queues.render.grow.reply.is_some() { return Err(EAGAIN); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 4713eb5b6b152..78e5e3a219358 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -6,7 +6,7 @@ use crate::g17::{ channel::Rings, - freelist::{Lease, Owner, RenderPool}, + freelist::{Lease, Owner, RenderPool, LINGER_NS}, fw::{ channels::{ControlRecord, FreeListCompletion}, initdata::PagePoolDescriptor, @@ -49,7 +49,7 @@ impl Pools { Ok(slot) } - fn owner(pool: &RenderPool) -> Owner { + pub(super) fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), epoch: pool.epoch(), @@ -154,12 +154,12 @@ impl Pools { } /// Preflight a paired retirement before QoS or PB reference accounting. - pub(super) fn completed(&self, id: u16, generation: u64) -> Result { + pub(super) fn completed(&self, id: u16, generation: u64, linger: bool) -> Result { let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; if next.generation() != generation { return Err(EIO); } - next.complete_render()?; + next.complete_render(linger)?; Ok(next) } @@ -169,6 +169,32 @@ impl Pools { self.leases[usize::from(id)] = Some(next); } + /// Ends the linger of every idle registration that `keep` (an open owner + /// still bound to the pool) does not retain or whose linger has elapsed. + /// Returns whether a release became pending, and the next expiry delay. + pub(super) fn expire_lingers( + &mut self, + now: u64, + keep: impl Fn(&Owner) -> bool, + ) -> (bool, Option) { + let mut ended = false; + let mut next: Option = None; + for lease in self.leases.iter_mut().flatten() { + let Some(since) = lease.lingering() else { + continue; + }; + let age = now.saturating_sub(since); + if age >= LINGER_NS || !keep(&lease.owner()) { + lease.end_linger(); + ended = true; + } else { + let delay = LINGER_NS - age; + next = Some(next.map_or(delay, |next| next.min(delay))); + } + } + (ended, next) + } + pub(super) fn pending(&self) -> bool { self.leases .iter() From b7fa0d482eea670f9004422eaf57703a7ec8d4e5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:26:24 -0400 Subject: [PATCH 27/64] drm/asahi: take the G17 device mutex less often to publish a render A render publication into an owner's installed pair took the device mutex six times from the scheduler (seven from the ioctl): to fetch the admission pool (and, from the ioctl, again to try it), to check the pair's identities, to find the pair already built (constructing a throwaway set of build resources), to look for a sibling pair to warm, to lend the pair out and to publish it. The admission pool lives as long as the firmware session, so the logical queue keeps a reference to it. The identity check also reports whether the owner's pair is installed, idle and backed, in which case the first pass skips graph construction; every retry still goes through it. An owner has exactly one pair (kick.rs asserts it), so there is no sibling to warm. Steady state now takes the device mutex three times from either path. gpubench unchanged within noise (render d1 151.6, d32 42.8, queues d8 58.8 us/submit); the mild stress gate passes. --- drivers/gpu/drm/asahi/g17/queue/admission.rs | 6 +++ drivers/gpu/drm/asahi/g17/runtime.rs | 4 ++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 22 ++++++++++ .../drm/asahi/g17/runtime/render/prepare.rs | 4 ++ .../drm/asahi/g17/runtime/render/submit.rs | 41 +++++++++---------- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index 424bb1b7601f5..b3cd061e0e912 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -686,6 +686,12 @@ mod accounting_tests { pub(crate) struct QueueSlot { pool: Arc, } +impl QueueSlot { + /// The device's admission pool, which lives as long as the firmware session. + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } +} impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 1bd500bae6cb0..30816cd1cbdb6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -664,6 +664,8 @@ pub(super) struct Backend { /// channel from being shared away between prepare_job and run_job. #[pin] claimed: kernel::sync::Mutex>>, + /// Render pair admission, shared by every logical queue of the session. + admission: Arc, } impl Backend { @@ -745,6 +747,7 @@ impl Backend { return Err(EAGAIN); } drop(state); + let admission = slot.pool().clone(); Ok(( Arc::pin_init( pin_init!(Self { @@ -756,6 +759,7 @@ impl Backend { closed: core::sync::atomic::AtomicBool::new(false), direct <- kernel::new_mutex!(None, "g17::Backend::direct"), claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), + admission, }), GFP_KERNEL, )?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index d770c433528c9..6360317b27851 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -326,6 +326,28 @@ impl Registry { Ok(None) } + /// The installed pair in `slot` belongs to `owner` and its context, is + /// neither borrowed nor settling a failure, and its pool has backing: the + /// state in which building the pair would do nothing. + pub(super) fn render_ready(&self, slot: u8, owner: u64, context: &Arc) -> Result { + let Ok(entry) = self.render.entry(slot) else { + return Ok(false); + }; + if entry.pending_error.is_some() + || entry.pending_timeout + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + { + return Ok(false); + } + let Some(pair) = entry.pair.as_ref() else { + return Ok(false); + }; + Ok(pair.matches(owner, context)? && !pair.pool().vacant()) + } + fn has_render_ids(&self, owner: u64) -> bool { self.render .entries diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 04afe2e05c107..62ca7895a86f8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -593,6 +593,10 @@ impl super::super::Backend { pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { self.build_render_pair(slot, packet, false, sleep)?; + // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. + if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { + return Ok(()); + } let sibling = { let state = self.shared.state.lock(); (*state) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index bd83c81b1d593..9d77a1e104328 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -44,16 +44,15 @@ impl Backend { /// Reserve identities before attaching a lease to the packet. An empty /// logical slot must not hide reusable physical channels when the shared /// queue-ID namespace is full. + /// Also reports whether the owner's installed pair is ready to take the + /// command, so that the publication skips graph construction. fn acquire_render_with_ids( &self, packet: &Arc, mut next: Option, sleep: bool, - ) -> Result { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; + ) -> Result<(RenderLease, bool)> { + let pool = &self.admission; let started = super::super::now_ns(); let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); // A capacity refusal samples the render domain and retries once; only @@ -108,7 +107,10 @@ impl Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; match firmware.queues.ensure_render_ids(slot, self.owner) { - Ok(()) => return Ok(lease), + Ok(()) => { + let ready = firmware.queues.render_ready(slot, self.owner, &self.context)?; + return Ok((lease, ready)); + } Err(error) if error == ENOSPC => { let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; if reclaimed.is_some() { @@ -282,7 +284,7 @@ impl Backend { // A publication from the submitting ioctl (an early lease) never // sleeps for contention; it falls back to the scheduler instead. let sleep = early.is_none(); - let admission = self.acquire_render_with_ids(packet, early, sleep)?; + let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; let mut busy_since = None; @@ -335,7 +337,14 @@ impl Backend { result?; } } - match self.ensure_render(packet, slot, sleep) { + // A pair found ready under the identity check needs no build; any + // retry revalidates through ensure_render. + let ensured = if core::mem::take(&mut ready) { + Ok(()) + } else { + self.ensure_render(packet, slot, sleep) + }; + match ensured { Err(error) if error == ENOSPC => { self.wait_render_capacity(&mut seen, capacity_started, sleep)?; continue; @@ -386,19 +395,9 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = { - let state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref() else { - return false; - }; - match firmware - .queues - .admission - .try_acquire_render(self.owner, Request::of(packet)) - { - Ok(Some(lease)) => lease, - _ => return false, - } + let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { + Ok(Some(lease)) => lease, + _ => return false, }; if !packet.begin_publication() { packet.completion.fail_unpublished(ECANCELED); From d272a7060d20a0beb635ca8426c8822b8e823a43 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:43:52 -0400 Subject: [PATCH 28/64] drm/asahi: publish a G17 render into its pair without the device mutex Every render publication held the device mutex, which the event worker also needs for each pass, across the whole publication: dependency and hazard collection, kick and descriptor commits, QoS accounting, both work-ring records and producers, and the doorbell. In the steady state (the owner's graph is installed and registered at its priority, last ran this owner's scheduler, needs no growth, has nothing to replay or settle, and the command has no compute prefix) the device mutex now only lends the pair out, together with its free-list preparation and a parameter-buffer reference, and takes it back. Preparation and publication in between take the port: a render publication lock that every render publication holds, with or without the device mutex (the tiling and fragment work rings keep one host producer at a time, and attachment overlaps keep their publication order), QoS accounting, the completion stamps, the doorbell and the full-effort witness. The port gains views of the render work rings and of the stamp page. The free-list lease leaves preparation, or is rolled back with the parameter-buffer reference, when the pair returns; while it is lent the event worker skips the pair, so no retirement can touch its commands or references. A recovery that starts during the publication fails it with EIO, as on the compute path. Anything else, including a first publication, growth and owner changes, still publishes under the device mutex. During gpubench render rows 306k of 320k publications took the new path. render.light d2/d8/d32 68.9/45.2/42.8 -> 65.9/43.5/41.1 us/submit, d1 150.2 (151.6), queues d8 58.7 (58.8); the rows stay firmware bound. Mild stress gate passes. --- drivers/gpu/drm/asahi/g17/initdata.rs | 25 +- drivers/gpu/drm/asahi/g17/port.rs | 125 +++++- drivers/gpu/drm/asahi/g17/queue/render.rs | 49 +++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 1 + .../drm/asahi/g17/runtime/render/direct.rs | 366 ++++++++++++++++++ .../drm/asahi/g17/runtime/render/submit.rs | 14 +- 6 files changed, 566 insertions(+), 14 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/direct.rs diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 6d0b2f9951956..8f7551b1763c9 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -659,15 +659,16 @@ impl InitData { Ok(()) } - /// Raw views of the memory publishers share without the device mutex: every compute work - /// ring, the QoS table and the recovery state word. They stay valid for this session. + /// Raw views of the memory publishers share without the device mutex: every work ring, + /// the completion stamps, the QoS table and the recovery state word. They stay valid for + /// this session. pub(super) fn port_views( &self, mailbox: *const super::Mailbox, ) -> Result { use super::fw::{channels, queue::{DataMaster, Policy}}; - let ring = |priority: u8| -> Result { - let channel = channels::work_ring_index(priority, DataMaster::Compute).ok_or(EINVAL)?; + let ring_of = |priority: u8, engine: DataMaster| -> Result { + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let word = |offset: usize| -> Result<*const AtomicU32> { @@ -686,12 +687,18 @@ impl InitData { producer: word(abi::ChannelState::PRODUCER)?, }) }; - let mut compute = [ring(0)?; Policy::PRIORITIES as usize]; - for (priority, view) in compute.iter_mut().enumerate().skip(1) { - *view = ring(priority as u8)?; - } + let rings = |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { + let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; + for (priority, view) in views.iter_mut().enumerate().skip(1) { + *view = ring_of(priority as u8, engine)?; + } + Ok(views) + }; Ok(super::port::Views { - compute, + compute: rings(DataMaster::Compute)?, + tiling: rings(DataMaster::Tiling)?, + fragment: rings(DataMaster::Fragment)?, + stamps: self.object(self.stamps[0])?.word(0)?, qos: self .qos()? .pointer(0, core::mem::size_of::())?, diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 63eab753632be..1adf9f9690456 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -7,7 +7,9 @@ //! through a preparation lease ([`super::Shared::port`]): shutdown and device loss join every //! lease before they detach the session. Each piece below is atomic or has its own lock. //! -//! Lock order: device mutex -> compute queue cell -> the locks of this port. +//! Lock order: device mutex -> compute queue cell -> the locks of this port. The render +//! publication lock comes first among the port's locks: device mutex -> render publication +//! lock -> admission pool, QoS, mailbox. Its holder never takes the device mutex. use super::{ fw::{ @@ -18,9 +20,9 @@ use super::{ }; use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; use kernel::{ - new_spinlock, + new_mutex, new_spinlock, prelude::*, - sync::{Arc, SetOnce, SpinLock}, + sync::{Arc, Mutex, MutexGuard, SetOnce, SpinLock}, }; const PRIORITIES: usize = Policy::PRIORITIES as usize; @@ -62,6 +64,10 @@ impl RingView { /// Raw views of the session's firmware-shared memory, fixed once the firmware is built. pub(super) struct Views { pub(super) compute: [RingView; PRIORITIES], + pub(super) tiling: [RingView; PRIORITIES], + pub(super) fragment: [RingView; PRIORITIES], + /// Completion stamp page A: one word per QID, written by the firmware. + pub(super) stamps: *const AtomicU32, pub(super) qos: *mut u8, pub(super) recovery: *const AtomicU32, pub(super) mailbox: *const super::Mailbox, @@ -81,6 +87,11 @@ pub(super) struct Port { rings: SpinLock<[u32; PRIORITIES]>, #[pin] qos: SpinLock, + /// Serializes every render publication, with or without the device mutex: the tiling + /// and fragment work rings have one host producer at a time, and same-VM attachment + /// overlaps are ordered by publication order (see hazard.rs). + #[pin] + render: Mutex<()>, /// The accepted recovery generation, mirrored for publishers without the device mutex. generation: AtomicU64, /// Device publication order shared by compute and render commands. @@ -114,6 +125,7 @@ impl Port { views: SetOnce::new(), rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), + render <- new_mutex!((), "g17::Port::render"), generation: AtomicU64::new(0), publications: AtomicU64::new(0), effort_full: AtomicBool::new(false), @@ -196,6 +208,113 @@ impl Port { Ok(next) } + /// Taken around every render publication; see `render`. + pub(super) fn render_lock(&self) -> MutexGuard<'_, ()> { + self.render.lock() + } + + fn render_rings(&self, priority: u8) -> Result<(RingView, RingView)> { + let views = self.views()?; + let index = usize::from(priority); + Ok(( + *views.fragment.get(index).ok_or(EINVAL)?, + *views.tiling.get(index).ok_or(EINVAL)?, + )) + } + + /// Both render work rings of `priority` have a free slot. Only firmware consumption, + /// never a publisher without the render lock, changes this. + pub(super) fn render_room(&self, _guard: &MutexGuard<'_, ()>, priority: u8) -> Result { + let (fragment, tiling) = self.render_rings(priority)?; + Ok(fragment.free()? > 0 && tiling.free()? > 0) + } + + /// Copies a render pair's fragment, then tiling record into the slots after each + /// producer without exposing them, as `InitData::publish_work_pair` does for a deferred + /// pair. Returns the producers that expose them. + pub(super) fn write_render_pair( + &self, + _guard: &MutexGuard<'_, ()>, + priority: u8, + fragment: &WorkSlot, + tiling: &WorkSlot, + commit_accounting: impl FnOnce() -> Result, + publish_late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let (fragment_ring, tiling_ring) = self.render_rings(priority)?; + let slot = |ring: &RingView| -> Result<(*mut u8, u32)> { + let [first, second, producer] = ring.cursors()?; + let next = (producer + 1) % WORK_RING_SLOTS; + if next == first || next == second { + return Err(EINVAL); + } + // SAFETY: The ring view spans WORK_RING_SLOTS records; `producer` is in range. + Ok((unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }, next)) + }; + let (fragment_slot, fragment_next) = slot(&fragment_ring)?; + let (tiling_slot, tiling_next) = slot(&tiling_ring)?; + let copy = |record: &WorkSlot, destination: *mut u8| { + // SAFETY: The slot after the producer is host-owned until the producer moves past + // it, and the render lock excludes every other host writer of this ring. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref(record).cast::(), + destination, + core::mem::size_of::(), + ) + } + }; + copy(fragment, fragment_slot); + commit_accounting()?; + fence(Ordering::SeqCst); + publish_late_tiling(); + copy(tiling, tiling_slot); + fence(Ordering::SeqCst); + Ok([fragment_next, tiling_next]) + } + + /// Exposes a pair written by `write_render_pair`, fragment first, as + /// `InitData::publish_retained_work_pair` does: the producers must still precede `next`. + pub(super) fn expose_render_pair( + &self, + _guard: &MutexGuard<'_, ()>, + priority: u8, + next: [u32; 2], + ) -> Result { + let (fragment_ring, tiling_ring) = self.render_rings(priority)?; + for (ring, next) in [(&fragment_ring, next[0]), (&tiling_ring, next[1])] { + let [_, _, producer] = ring.cursors()?; + if next != (producer + 1) % WORK_RING_SLOTS { + return Err(EBUSY); + } + } + fence(Ordering::SeqCst); + // SAFETY: As for `RingView::cursors`. + unsafe { (*fragment_ring.producer).store(next[0], Ordering::Relaxed) }; + fence(Ordering::SeqCst); + // SAFETY: As above. + unsafe { (*tiling_ring.producer).store(next[1], Ordering::Relaxed) }; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The completion stamp of `qid` in page A. + pub(super) fn stamp(&self, qid: u8) -> Result { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let base = self.views()?.stamps; + // SAFETY: Page A holds one word per QID up to QID_MAX; the session retains it. + let value = unsafe { (*base.add(usize::from(qid))).load(Ordering::Relaxed) }; + fence(Ordering::Acquire); + Ok(value) + } + fn qos_view(&self) -> Result> { // SAFETY: The session retains the QoS table for at least as long as this port. unsafe { qos::View::from_raw(self.views()?.qos) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index faf31514a2f2f..3e65ea3b9a367 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1787,6 +1787,55 @@ impl Pair { } /// The scheduler page these queues last ran a command for, when the next /// publication for `context` changes the queues' owner. + /// The steady state in which a command may publish without the device + /// mutex: the graph is installed and registered at `priority`, it last ran + /// `context`'s scheduler, and nothing is replayed, settled, aliased or + /// switched on it. + pub(crate) fn direct_ready(&self, context: &Context, priority: u8) -> bool { + self.graph_started + && self.installed == [true; 2] + && self.registered == Some(priority) + && priority < 8 + && self.outer_started & (1u8 << priority) != 0 + && self.executed == context.scheduler_va() + && self.switch.is_none() + && self.owner.is_some() + && !self.quarantined + && self.terminal.is_none() + && self.pending_aliases.is_none() + && self.kick_aliases.is_none() + && self.prepared.is_none() + && self.replay.len() == 0 + && self.displaced.is_none() + && self.active.room() + } + /// The render prefix of `packet` on this pair, as the registry collects it + /// for an owner whose only pair this is. + pub(crate) fn own_dependencies(&self, owner: u64, packet: &Arc) -> Result { + let mut dependencies = RenderDependencies::EMPTY; + let mut frontier = crate::g17::dependency::Frontier::new(); + self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier)?; + for dependency in frontier.as_slice() { + *dependencies.render.get_mut(dependencies.render_count).ok_or(EOVERFLOW)? = *dependency; + dependencies.render_count += 1; + } + if dependencies.render_count != 0 + && matches!(&packet.command, Validated::Render { pass, .. } if pass.fragment_barriers) + { + let mut tiling = crate::g17::dependency::Frontier::new(); + self.collect_tiling_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut tiling, + )?; + for dependency in tiling.as_slice() { + *dependencies.tiling.get_mut(dependencies.tiling_count).ok_or(EOVERFLOW)? = *dependency; + dependencies.tiling_count += 1; + } + } + Ok(dependencies) + } pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 6360317b27851..aa437b8a21be2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -6,6 +6,7 @@ mod exit; mod dependencies; +mod direct; mod grow; mod observe; mod pools; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs new file mode 100644 index 0000000000000..00e0037ebae60 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Steady-state render publication without the device mutex. +//! +//! The device mutex lends the owner's installed pair out, together with its +//! free-list preparation and a parameter-buffer reference, and takes it back. +//! Between the two, command preparation and publication take only the port: +//! its render publication lock (the render work rings' single producer and +//! the publication order of attachment hazards), QoS accounting, the stamps +//! and the doorbell. While the pair is lent, the event worker skips it, so no +//! retirement can complete its commands or their free-list references. + +use super::super::{Backend, DeferredBatch}; +use crate::g17::{ + job::Packet, + object::Allocator, + port::Port, + queue::render::{Pair, PublicationLease, RenderAddresses}, +}; +use core::sync::atomic::Ordering; +use kernel::{ + prelude::*, + sync::{Arc, MutexGuard}, +}; + +/// A pair lent out for one publication, with what that publication holds. +pub(in crate::g17::runtime) struct Loan { + pair: Option>, + preparation: Option, + buffer: u8, + lease: PublicationLease, + addresses: RenderAddresses, + uat: Arc, +} + +/// Publication host of a lent pair. Registration, first-publication and +/// growth paths never run here: the loan requires an installed, registered +/// graph that needs no growth. +struct DirectHost<'a> { + port: &'a Port, + render: &'a MutexGuard<'a, ()>, + generation: u64, + activation: Option<(u16, u64)>, +} + +impl crate::g17::queue::render::Host for DirectHost<'_> { + fn next_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { + Ok([self.port.stamp(qids[0])?, self.port.stamp(qids[1])?]) + } + fn qos_publish( + &mut self, + owner: crate::g17::qos::Owner, + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result { + self.port.qos_publish(owner, scheduler, policy) + } + fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn install_pair(&mut self, _registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + Err(EAGAIN) + } + fn publish_qid(&mut self, _id: crate::g17::kick::Id) -> Result { + // An installed graph published its identities with its first command. + Ok(()) + } + fn publish_outer_pair( + &mut self, + priority: u8, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + self.port + .write_render_pair(self.render, priority, fragment, tiling, commit, late_tiling) + } + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { + // The lease leaves preparation when the pair is returned; nothing can + // observe it in between because the pair is lent. + self.activation = Some((id, generation)); + Ok(()) + } + fn publish_retained_pair( + &mut self, + priority: u8, + _fragment: &crate::g17::fw::channels::WorkSlot, + _tiling: &crate::g17::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + self.port.expose_render_pair(self.render, priority, next) + } + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + fn notify_render(&mut self, priority: u8) -> Result { + self.port + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2) + } + fn note_submission(&mut self) -> Result { + // Counted, with full effort witnessed, before the pair was lent. + Ok(()) + } + fn complete_render( + &mut self, + _owners: Option<[crate::g17::qos::Owner; 2]>, + _pool_id: u16, + _generation: u64, + _buffer_id: u8, + ) -> Result { + Err(EINVAL) + } +} + +impl crate::g17::Firmware { + /// Lends `owner`'s pair in `slot` for a publication of `packet`, or None if + /// the pair is not in the steady state the direct path requires. + fn lend_render( + &mut self, + slot: u8, + owner: u64, + packet: &Arc, + priority: u8, + generation: u64, + ) -> Result> { + if self.recovery.pending() || self.recovery.generation() != generation { + return Ok(None); + } + let Ok(entry) = self.queues.render.entry_mut(slot) else { + return Ok(None); + }; + if !entry.published + || entry.closed + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + || entry.pending_error.is_some() + || entry.pending_timeout + { + return Ok(None); + } + let Some(pair) = entry.pair.as_mut() else { + return Ok(None); + }; + if !pair.matches(owner, &packet.context)? + || !pair.direct_ready(&packet.context, priority) + || pair.pool().vacant() + || !matches!(pair.host_growth_target(packet), Ok(None)) + { + return Ok(None); + } + let qids = pair.qids(); + let pool = pair.pool().clone(); + let buffer_id = pair.buffer_id(); + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let (buffer, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { + Ok(preparation) => preparation, + Err(error) => { + self.queues.buffers.release(buffer)?; + return if error == EAGAIN || error == EBUSY { + Ok(None) + } else { + Err(error) + }; + } + }; + let lease = PublicationLease { + free_list_generation: preparation.generation, + buffer_token, + }; + let pair = match self.queues.render.take_pair(slot, owner, packet) { + Ok(pair) => pair, + Err(error) => { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(buffer)?; + return if error == EBUSY { Ok(None) } else { Err(error) }; + } + }; + // The selected scheduler is retained through release acknowledgement, + // as on the device-mutex path. + packet.context.mark_published(); + Ok(Some(Loan { + pair: Some(pair), + preparation: Some(preparation), + buffer, + lease, + addresses, + uat: self.uat.clone(), + })) + } + + /// Takes a lent pair back and settles what its publication held: the + /// free-list preparation is activated once published, rolled back with the + /// parameter-buffer reference otherwise. + fn return_render( + &mut self, + slot: u8, + loan: &mut Loan, + activation: Option<(u16, u64)>, + packet: &Arc, + ) -> Result { + self.queues.render.return_pair(slot, &mut loan.pair)?; + if self.queues.render.take_missed_event(slot) { + self.primary.state.shared.queue_events(); + } + let entry = self.queues.render.entry_mut(slot)?; + // A failure or timeout recorded against the lent pair is settled by the + // event worker's collector, as after any other loan. + if entry.pending_error.is_some() || entry.pending_timeout { + self.primary.state.shared.queue_events(); + } + let pair = entry.pair.as_mut().ok_or(EIO)?; + if entry.closed { + pair.release_owner(); + } + if !packet.is_published() { + pair.discard_prepared(packet)?; + } + if let Some((id, generation)) = activation { + if self.queues.render.pools.activate(id, generation).is_err() { + self.primary.state.shared.lose_device_quietly(); + } + } + let preparation = loan.preparation.take().ok_or(EIO)?; + if packet.is_published() { + self.queues.render.entry_mut(slot)?.published = true; + } else { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(loan.buffer)?; + } + Ok(()) + } +} + +impl Backend { + /// Publishes `packet` into the owner's installed pair with the device + /// mutex held only to lend the pair out and take it back. None leaves the + /// packet unpublished for the device-mutex path. + pub(in crate::g17::runtime) fn render_direct( + &self, + packet: &Arc, + slot: u8, + deferred: &mut DeferredBatch, + ) -> Option { + // A compute prefix needs the compute queue's state; leave it to the + // device-mutex path. + if packet.is_published() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || packet.order.wait_through[1].unwrap_or(0) != 0 + || self.admission_open().is_err() + { + return None; + } + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire() { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let priority = packet.context.policy().priority(); + let generation = port.generation(); + let dev = self.shared.drm().ok()?; + let mut loan = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut()?; + if !lease.is_current() { + return None; + } + match firmware.lend_render(slot, self.owner, packet, priority, generation) { + Ok(Some(loan)) => loan, + Ok(None) => return None, + Err(error) => return Some(Err(error)), + } + }; + let mut activation = None; + let result = (|| { + let pair = loan.pair.as_mut().ok_or(EIO)?; + let alloc = Allocator { + dev: &dev, + uat: &loan.uat, + }; + pair.prepare(&alloc, packet, loan.addresses)?; + let render = port.render_lock(); + if !lease.is_current() || !port.render_room(&render, priority)? { + return Err(EBUSY); + } + let mut dependencies = pair.own_dependencies(self.owner, packet)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.admission.publication_barriers( + self.owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } + let mut host = DirectHost { + port, + render: &render, + generation, + activation: None, + }; + let result = pair.publish( + packet.clone(), + &dependencies, + loan.lease, + &mut host, + &mut |event| deferred.push(event), + ); + activation = host.activation; + result?; + // A recovery that started meanwhile may have discarded the commands. + if port.generation() != generation || port.recovery_state()? != 0 { + return Err(EIO); + } + Ok(()) + })(); + let returned = { + let mut state = self.shared.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) => { + let returned = firmware.return_render(slot, &mut loan, activation, packet); + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { + self.shared.queue_grow(); + } + returned + } + // Shutdown joins every preparation lease before detaching the + // session, so this cannot happen while `lease` is held. + None => Err(ENODEV), + } + }; + self.shared.note_changed(); + drop(lease); + if let Err(error) = returned { + self.shared.lose_device_quietly(); + return Some(Err(error)); + } + match result { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => Some(result), + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 9d77a1e104328..a9dac3dab2f73 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -287,6 +287,11 @@ impl Backend { let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; + if ready { + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + } let mut busy_since = None; let capacity_started = super::super::now_ns(); let deadline_ns = capacity_started.saturating_add(super::super::WORK_RING_WAIT_NS); @@ -549,15 +554,20 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } + let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; + // Every render publication, with or without the device mutex, holds the + // port's render lock: the render work rings have one host producer at a + // time and attachment overlaps are ordered by publication order. + let port = self.port.clone(); + let _render = port.render_lock(); // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. All host producers share this lock. + // either inner queue becomes visible. let priority = packet.context.policy().priority(); if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } - let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; if let Some(footprint) = packet.footprint.as_ref() { dependencies.hazard_count = self.queues.admission.publication_barriers( owner, From 6428f7fd42fe075981984da569b0fa076f59a127 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:52:57 -0400 Subject: [PATCH 29/64] drm/asahi: lend a G17 render pair without the identity check first After the direct render path, a steady-state publication still took the device mutex three times: once to check the pair's identities and readiness, then to lend the pair and to take it back. The lend itself validates everything the identity check does. The logical queue remembers the pair it last published to directly. A lease on that pair (the ioctl's, or a non-blocking admission from the scheduler) goes straight to the lend; anything else, and any lend that declines, takes the full path with the identity check. gpubench unchanged within noise (render d1 150.8, d32 42.9, queues d8 58.4 us/submit); the mild stress gate passes. --- drivers/gpu/drm/asahi/g17/runtime.rs | 4 +++ .../drm/asahi/g17/runtime/render/direct.rs | 7 +++++- .../drm/asahi/g17/runtime/render/submit.rs | 25 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 30816cd1cbdb6..b07423bd22115 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -666,6 +666,9 @@ pub(super) struct Backend { claimed: kernel::sync::Mutex>>, /// Render pair admission, shared by every logical queue of the session. admission: Arc, + /// The render pair this owner last published to without the device mutex + /// (zero for none): a hint, revalidated by every lend. + direct_slot: core::sync::atomic::AtomicU32, } impl Backend { @@ -760,6 +763,7 @@ impl Backend { direct <- kernel::new_mutex!(None, "g17::Backend::direct"), claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), admission, + direct_slot: core::sync::atomic::AtomicU32::new(0), }), GFP_KERNEL, )?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index 00e0037ebae60..cd7bad6ecce5d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -360,7 +360,12 @@ impl Backend { } match result { Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, - result => Some(result), + result => { + if result.is_ok() { + self.direct_slot.store(u32::from(slot), Ordering::Relaxed); + } + Some(result) + } } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index a9dac3dab2f73..733a570f0e861 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -284,6 +284,31 @@ impl Backend { // A publication from the submitting ioctl (an early lease) never // sleeps for contention; it falls back to the scheduler instead. let sleep = early.is_none(); + let mut early = early; + // The pair this owner last published to directly: a lease on it goes + // straight to the direct path, whose lend checks everything the + // identity check would. + let known = self.direct_slot.load(Ordering::Relaxed); + let lease = match early.take() { + Some(lease) => Some(lease), + None if known != 0 => self + .admission + .try_acquire_render(self.owner, Request::of(packet)) + .ok() + .flatten(), + None => None, + }; + let early = match lease { + Some(lease) if u32::from(lease.slot()) == known => { + let slot = lease.slot(); + packet.completion.attach_render_lease(lease)?; + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + packet.completion.detach_render_lease() + } + lease => lease, + }; let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; From 8cf02f589bce34b13002cbdf0f3b0361c4a8e66b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 01:04:35 -0400 Subject: [PATCH 30/64] drm/asahi: wait for G17 render admission before run_job The scheduler's prepare step returned no fence for render jobs, so a job that could not get a work state or a pair admission (a cross-VM attachment overlap, a crowded VM, no pair capacity) waited for it inside run_job, holding the scheduler worker. prepare_job now checks both without taking anything: a work state of the context, and whether the admission pool would admit the command (its owner's pair has room and no overlap plan says wait, or a free pair exists for an owner without one). When either is missing it returns a fence of the domain whose return provides it: the context's work-state domain, or the render domain that every admission-pool return signals. The scheduler calls prepare again once it fires. Each check runs once more after sampling its domain, so the common case takes no domain lock. gpubench unchanged within noise (render d1 151.8, d32 42.7, queues d8 58.8 us/submit); the mild stress gate passes. --- drivers/gpu/drm/asahi/g17/queue/admission.rs | 40 +++++++++++++++ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 5 +- .../drm/asahi/g17/runtime/render/submit.rs | 49 +++++++++++++++++++ 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index b3cd061e0e912..e1e7552c243f2 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -298,6 +298,28 @@ impl State { ) } + /// Whether `pick` would return a slot, without taking it. + fn can_pick(&self, owner: u64, plan: Plan) -> bool { + let usable = |slot: usize| { + self.owners[slot] == owner + && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 + && self.refs[slot] < cfg::RENDER_DEPTH + }; + match plan { + Plan::Wait => false, + Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), + Plan::Any if owner == 0 => false, + Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), + Plan::Any => { + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + (0..cfg::RENDER_SLOTS).any(|slot| { + free & (1u64 << slot) != 0 + && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) + }) + } + } + } + fn pick(&mut self, owner: u64, plan: Plan) -> Option { match plan { Plan::Any => self.acquire(owner), @@ -555,6 +577,24 @@ impl Pool { record, })) } + /// Whether `acquire_render` would admit this request now, without + /// admitting it: the scheduler's prepare step waits for this. + pub(crate) fn admits_render(&self, owner: u64, request: Option>) -> bool { + let state = self.state.lock(); + if owner == 0 || state.refusal(owner).is_err() { + return false; + } + if state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) + { + return false; + } + let plan = state.plan(owner, request, None); + state.can_pick(owner, plan) + } pub(crate) fn warmable_sibling( self: &Arc, owner: u64, diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 39f628d007e18..d19f51c08dd29 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -216,11 +216,12 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - /// Compute admission waits for its shared capacity here, as a fence. + /// Compute and render admission wait for their shared capacity here, as a + /// fence. fn prepare(&self, packet: &Arc) -> Option { match packet.engine() { Engine::Compute => self.prepare_compute(packet), - Engine::Render => None, + Engine::Render => self.admit_render(packet), } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 733a570f0e861..1ef6742b689a7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -141,6 +141,55 @@ impl Backend { } } + /// Scheduler admission of a render packet. A work state of its context or + /// a pair admission it lacks becomes a fence that the return providing it + /// signals, so run_job finds both ready instead of waiting for them. + /// Errors, and anything else publication may wait for, are left to it. + pub(in crate::g17::runtime) fn admit_render(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + // Each check runs once more after sampling its domain, so the common + // case takes no domain lock and no return between check and wait is lost. + let work = packet.completion.work_state().ok()?; + loop { + match work.acquire() { + Err(error) if error == EBUSY => { + let states = work.returns().observe(); + match work.acquire() { + Err(error) if error == EBUSY => match work.returns().wait(states) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + }, + Err(_) => return None, + Ok(_) => {} + } + } + Err(_) => return None, + Ok(_) => {} + } + let request = Request::of(packet); + if self.admission.admits_render(self.owner, request) { + return None; + } + let observed = self.shared.renders.observe(); + if self.admission.admits_render(self.owner, request) { + return None; + } + self.shared.queue_events(); + match self.shared.renders.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + /// Conditions which end a publication that has not become visible yet. fn publication_open(&self, packet: &Packet) -> Result { self.admission_open()?; From 550fb355e9a30c0db9c59fe35a931f1db9c6b5c4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:16:03 -0400 Subject: [PATCH 31/64] drm/asahi: trace the G17 core power state at each work doorbell Add an asahi_g17_doorbell tracepoint that samples the core power register just before a compute or render work doorbell rings. The register is read only while the event is enabled, so tracing can tell whether a slow doorbell-to-start interval coincides with powered-down cores. --- drivers/gpu/drm/asahi/g17/mod.rs | 2 +- drivers/gpu/drm/asahi/g17/port.rs | 10 ++++++++-- drivers/gpu/drm/asahi/g17/runtime/render.rs | 3 +++ drivers/gpu/drm/asahi/g17/trace.c | 10 ++++++++++ drivers/gpu/drm/asahi/g17/trace.h | 17 +++++++++++++++++ drivers/gpu/drm/asahi/g17/trace.rs | 13 +++++++++++++ drivers/gpu/drm/asahi/regs.rs | 5 +++++ 7 files changed, 57 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 5e4c0dcc756cb..96a0991cfd143 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1361,7 +1361,7 @@ impl Gpu { session.uat.confirm_firmware_root_entries(root_entries)?; } let mailbox: *const Mailbox = &*session.primary.rtkit; - session.port.attach(session.init.port_views(mailbox)?)?; + session.port.attach(session.init.port_views(mailbox)?, session.bridge.regs.clone())?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 1adf9f9690456..146b83972f203 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -82,6 +82,8 @@ unsafe impl Sync for Views {} #[pin_data] pub(super) struct Port { views: SetOnce, + /// Register access for tracing the core power state at the doorbell. + regs: SetOnce, /// Compute work-ring slots reserved by publishers that have not exposed them yet. #[pin] rings: SpinLock<[u32; PRIORITIES]>, @@ -123,6 +125,7 @@ impl Port { Arc::pin_init( pin_init!(Self { views: SetOnce::new(), + regs: SetOnce::new(), rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), render <- new_mutex!((), "g17::Port::render"), @@ -136,8 +139,8 @@ impl Port { } /// Called once while the session is built, before any publisher can see the port. - pub(super) fn attach(&self, views: Views) -> Result { - if self.views.populate(views) { + pub(super) fn attach(&self, views: Views, regs: crate::regs::G17Resources) -> Result { + if self.views.populate(views) && self.regs.populate(regs) { Ok(()) } else { Err(EBUSY) @@ -344,6 +347,9 @@ impl Port { pub(super) fn notify(&self, message: u64) -> Result { // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port. let mailbox = unsafe { &*self.views()?.mailbox }; + super::trace::doorbell(super::trace::COMPUTE, || { + self.regs.as_ref().map_or(0, |regs| regs.g17_core_power()) + }); mailbox.send(super::EP_DOORBELL, message) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index aa437b8a21be2..f4f45a98005c3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -612,6 +612,9 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); + crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { + self.firmware.bridge.regs.g17_core_power() + }); let result = self .firmware .primary diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c index feffb96315845..fc189458af32d 100644 --- a/drivers/gpu/drm/asahi/g17/trace.c +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -31,6 +31,16 @@ void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind) trace_asahi_g17_bind(engine, qid, ctx, kind); } +bool asahi_g17_trace_doorbell_enabled(void) +{ + return trace_asahi_g17_doorbell_enabled(); +} + +void asahi_g17_trace_doorbell(u8 engine, u32 power) +{ + trace_asahi_g17_doorbell(engine, power, __arch_counter_get_cntvct()); +} + void asahi_g17_trace_event(u8 phase) { if (trace_asahi_g17_event_enabled()) diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h index e1ef55715e312..e00271249ce64 100644 --- a/drivers/gpu/drm/asahi/g17/trace.h +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -91,6 +91,23 @@ TRACE_EVENT(asahi_g17_bind, __entry->engine, __entry->qid, __entry->ctx, __entry->kind) ); +TRACE_EVENT(asahi_g17_doorbell, + TP_PROTO(u8 engine, u32 power, u64 counter), + TP_ARGS(engine, power, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u32, power) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->power = power; + __entry->counter = counter; + ), + TP_printk("engine=%u power=%#x counter=%llu", __entry->engine, __entry->power, + __entry->counter) +); + TRACE_EVENT(asahi_g17_event, TP_PROTO(u8 phase, u64 counter), TP_ARGS(phase, counter), diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs index a616b68735225..d0150c7318958 100644 --- a/drivers/gpu/drm/asahi/g17/trace.rs +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -12,6 +12,8 @@ unsafe extern "C" { fn asahi_g17_trace_complete(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, gpu_start: u64, gpu_end: u64); fn asahi_g17_trace_bind(engine: u8, qid: u8, ctx: u32, kind: u8); + fn asahi_g17_trace_doorbell_enabled() -> bool; + fn asahi_g17_trace_doorbell(engine: u8, power: u32); fn asahi_g17_trace_event(phase: u8); } @@ -49,6 +51,17 @@ pub(super) fn bind(engine: u8, qid: u8, ctx: u32, kind: u8) { unsafe { asahi_g17_trace_bind(engine, qid, ctx, kind) }; } +/// A work doorbell is about to ring; `power` samples the core power state +/// only while the event is enabled. +pub(super) fn doorbell(engine: u8, power: impl FnOnce() -> u32) { + // SAFETY: Pure static-key query. + if !unsafe { asahi_g17_trace_doorbell_enabled() } { + return; + } + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_doorbell(engine, power()) }; +} + /// Event worker pass boundaries: 0 begin, 1 fences signalled, 2 end. pub(super) fn event(phase: u8) { // SAFETY: Plain values only. diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index e7f6fd810b4ad..e8b845cba989f 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -709,6 +709,11 @@ impl G17Resources { self.sgx_read32::() & 0xf != 0 } + /// Raw core power state, for tracing only. + pub(crate) fn g17_core_power(&self) -> u32 { + self.sgx_read32::() + } + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { From f381ac9b71ebcb70eb72497ec4115553856a31fc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 00:32:18 -0400 Subject: [PATCH 32/64] drm/asahi: trace G17 host-to-firmware messages Add an asahi_g17_message tracepoint for every mailbox message, device-control record (by opcode) and effort update the host sends to the firmware, so a trace can show what the firmware was handling when a doorbell was slow to start. --- drivers/gpu/drm/asahi/g17/channel.rs | 1 + drivers/gpu/drm/asahi/g17/mod.rs | 2 ++ drivers/gpu/drm/asahi/g17/port.rs | 1 + drivers/gpu/drm/asahi/g17/trace.c | 6 ++++++ drivers/gpu/drm/asahi/g17/trace.h | 17 +++++++++++++++++ drivers/gpu/drm/asahi/g17/trace.rs | 12 ++++++++++++ 6 files changed, 39 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 8e4740b76a166..d96924cbbf9f9 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -138,6 +138,7 @@ impl Rings { record.opcode().asserts_gpu_power(), next, )?; + super::trace::message(super::trace::CONTROL, record.opcode() as u64); Ok(next) } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 96a0991cfd143..0e29757a0c2cc 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -548,6 +548,7 @@ impl Coprocessor { /// Sends a host notification to the firmware. fn notify(&self, msg: u64) -> Result { + trace::message(trace::MAILBOX, msg); self.rtkit.send(EP_DOORBELL, msg) } @@ -706,6 +707,7 @@ impl Firmware { return Ok(()); } init.publish_effort(effort)?; + trace::message(trace::EFFORT, u64::from(effort)); state.did_apply(effort); port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 146b83972f203..d8f7f4ddff8a1 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -350,6 +350,7 @@ impl Port { super::trace::doorbell(super::trace::COMPUTE, || { self.regs.as_ref().map_or(0, |regs| regs.g17_core_power()) }); + super::trace::message(super::trace::MAILBOX, message); mailbox.send(super::EP_DOORBELL, message) } diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c index fc189458af32d..ae977c13aab3b 100644 --- a/drivers/gpu/drm/asahi/g17/trace.c +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -41,6 +41,12 @@ void asahi_g17_trace_doorbell(u8 engine, u32 power) trace_asahi_g17_doorbell(engine, power, __arch_counter_get_cntvct()); } +void asahi_g17_trace_message(u8 kind, u64 value) +{ + if (trace_asahi_g17_message_enabled()) + trace_asahi_g17_message(kind, value, __arch_counter_get_cntvct()); +} + void asahi_g17_trace_event(u8 phase) { if (trace_asahi_g17_event_enabled()) diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h index e00271249ce64..7e5a50e774b59 100644 --- a/drivers/gpu/drm/asahi/g17/trace.h +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -108,6 +108,23 @@ TRACE_EVENT(asahi_g17_doorbell, __entry->counter) ); +TRACE_EVENT(asahi_g17_message, + TP_PROTO(u8 kind, u64 value, u64 counter), + TP_ARGS(kind, value, counter), + TP_STRUCT__entry( + __field(u8, kind) + __field(u64, value) + __field(u64, counter) + ), + TP_fast_assign( + __entry->kind = kind; + __entry->value = value; + __entry->counter = counter; + ), + TP_printk("kind=%u value=%#llx counter=%llu", __entry->kind, __entry->value, + __entry->counter) +); + TRACE_EVENT(asahi_g17_event, TP_PROTO(u8 phase, u64 counter), TP_ARGS(phase, counter), diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs index d0150c7318958..57db5abaccef8 100644 --- a/drivers/gpu/drm/asahi/g17/trace.rs +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -12,6 +12,7 @@ unsafe extern "C" { fn asahi_g17_trace_complete(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, gpu_start: u64, gpu_end: u64); fn asahi_g17_trace_bind(engine: u8, qid: u8, ctx: u32, kind: u8); + fn asahi_g17_trace_message(kind: u8, value: u64); fn asahi_g17_trace_doorbell_enabled() -> bool; fn asahi_g17_trace_doorbell(engine: u8, power: u32); fn asahi_g17_trace_event(phase: u8); @@ -62,6 +63,17 @@ pub(super) fn doorbell(engine: u8, power: impl FnOnce() -> u32) { unsafe { asahi_g17_trace_doorbell(engine, power()) }; } +pub(super) const MAILBOX: u8 = 0; +pub(super) const CONTROL: u8 = 1; +pub(super) const EFFORT: u8 = 2; + +/// A host-to-firmware message: a mailbox message, a device-control record by +/// opcode, or a new performance effort value. +pub(super) fn message(kind: u8, value: u64) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_message(kind, value) }; +} + /// Event worker pass boundaries: 0 begin, 1 fences signalled, 2 end. pub(super) fn event(phase: u8) { // SAFETY: Plain values only. From adcd56e020362664c7bc3af288e1ad812df7b4c7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 02:09:59 -0400 Subject: [PATCH 33/64] drm/asahi: maintain cumulative completion scopes without repeated fence merges Publish queue-scoped completion outputs with bounded callback bookkeeping while preserving independent GPU execution and native input dependencies. Signal ready scopes iteratively, propagate errors and deadline hints, and release completed history. Validated on Neo with the full benchmark and stress gate, published software-fence ordering, 256 queued completions, deadline forwarding and injected errors. Same-boot compute submission CPU drops from about 14 us to 10 us; GPU throughput remains near 20 us. --- drivers/gpu/drm/asahi/Makefile | 2 +- drivers/gpu/drm/asahi/file.rs | 45 +++- drivers/gpu/drm/asahi/g17/fence.rs | 42 ++- drivers/gpu/drm/asahi/g17/mod.rs | 4 +- drivers/gpu/drm/asahi/g17/sync_input.c | 72 ++++-- drivers/gpu/drm/asahi/g17/sync_output.c | 138 ++++++++++ drivers/gpu/drm/asahi/g17/sync_output.h | 23 ++ drivers/gpu/drm/asahi/g17/sync_output.rs | 69 +++++ drivers/gpu/drm/asahi/g17/sync_scope.c | 315 +++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/sync_scope.h | 15 ++ include/uapi/drm/asahi_drm.h | 17 ++ rust/kernel/drm/syncobj.rs | 8 + 12 files changed, 719 insertions(+), 31 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/sync_output.c create mode 100644 drivers/gpu/drm/asahi/g17/sync_output.h create mode 100644 drivers/gpu/drm/asahi/g17/sync_output.rs create mode 100644 drivers/gpu/drm/asahi/g17/sync_scope.c create mode 100644 drivers/gpu/drm/asahi/g17/sync_scope.h diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 9edbc6ba3b8d4..6d15b73fc4739 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/trace.o +asahi-y := asahi_core.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G16) += g16/glue.o diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index d1b8084902bcc..9c662ab6b85a2 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -127,10 +127,18 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, + pub(crate) cumulative: Option, + pub(crate) timeline: bool, } impl SyncItem { - fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + fn parse_one(file: &DrmFile, mut data: uapi::drm_asahi_sync, out: bool, + bundled: bool) -> Result + { + if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !out || !bundled || data.timeline_value == 0 { return Err(EINVAL); } + data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; + } match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -151,6 +159,8 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -173,12 +183,10 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: if out { - Some(dma_fence::FenceChain::new()?) - } else { - None - }, + chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: true, }) } _ => { @@ -193,12 +201,14 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, + allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let mut anchor = None; // SAFETY: We only read this once, so there are no TOCTOU issues. let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); @@ -214,7 +224,25 @@ impl SyncItem { // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !allow_cumulative || i < in_count || anchor.is_some() { + return Err(EINVAL); + } + anchor = Some(i as usize); + } + vec.push(Self::parse_one(file, sync, i >= in_count, + allow_cumulative)?, GFP_KERNEL)?; + } + if let Some(anchor) = anchor { + let bundle = crate::g17::sync_output::SignalBundle::new( + &vec[in_count as usize..], anchor - in_count as usize)?; + vec[anchor].cumulative = Some(bundle); + } else { + for sync in &mut vec[in_count as usize..] { + if sync.timeline { + sync.chain_fence = Some(dma_fence::FenceChain::new()?); + } + } } Ok(vec) @@ -1396,7 +1424,8 @@ impl File { copy_commands(&mut vec)?; } let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count, + gpu.as_any().is::())?; mod_dev_dbg!( device, diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index da883cede52df..8245e8bca71e6 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -19,6 +19,16 @@ use kernel::{ pub(crate) struct CompletionFence; +unsafe extern "C" { + fn asahi_g17_signal_fence(fence: *mut bindings::dma_fence); +} + +fn signal_exported(fence: &dyn RawDmaFence) { + // SAFETY: The borrowed fence remains owned throughout synchronous signaling. + // Ready completion scopes are drained iteratively after its lock is released. + unsafe { asahi_g17_signal_fence(fence.raw()) }; +} + #[vtable] impl FenceOps for CompletionFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -52,6 +62,7 @@ pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { /// Submission never owns this fence, so exported references cannot form a cycle. pub(crate) struct ExportFence { source: Option>, + scoped: bool, } static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); @@ -65,7 +76,7 @@ impl FenceOps for ExportFence { #[inline(never)] fn export_fallback(contexts: &FenceContexts) -> Result> { - let fence = contexts.new_fence(0, ExportFence { source: None })?; + let fence = contexts.new_fence(0, ExportFence { source: None, scoped: false })?; // SAFETY: Unique, initialized and still private. Only this constructor // publishes ExportFence objects; its payload is immutable after sharing. let ops = unsafe { @@ -179,7 +190,7 @@ impl Submission { let error = self.error.load(Ordering::Acquire); if error != 0 { fence.set_error(Error::from_errno(error)); } else { self.release_state(); } - fence.signal(); + if fence.scoped { signal_exported(fence); } else { fence.signal(); } } } } @@ -332,30 +343,42 @@ pub(crate) struct Outputs { input_count: usize, submission: Option, result: Result, + cumulative: Option, } impl Outputs { pub(crate) fn new(contexts: &FenceContexts, mut syncs: KVec, input_count: usize, allocation_failure: &Fence) -> Result { if input_count > syncs.len() { return Err(EINVAL); } + let mut cumulative = syncs.iter_mut().find_map(|sync| sync.cumulative.take()); let fallback = match export_fallback(contexts) { Ok(fence) => fence, Err(error) => { - install(syncs.drain(input_count..), allocation_failure); + if let Some(bundle) = &mut cumulative { + bundle.fail(allocation_failure); + } else { + install(syncs.drain(input_count..), allocation_failure); + } return Err(error); } }; Ok(Self { fallback: Some(fallback), syncs, input_count, - submission: None, result: Err(ECANCELED) }) + submission: None, result: Err(ECANCELED), cumulative }) } pub(crate) fn inputs(&self) -> &[SyncItem] { &self.syncs[..self.input_count] } pub(crate) fn publish(&mut self, context: Arc, firmware_source: bool) -> Result { let source = Submission::new(context, firmware_source)?; let mut fence = self.fallback.take().ok_or(EINVAL)?; fence.source = Some(source); + fence.scoped = self.cumulative.is_some(); let submission = Aggregate(fence.into()); - install(self.syncs.drain(self.input_count..), &submission.fence()); self.submission = Some(submission.clone()); + if let Some(bundle) = &mut self.cumulative { + bundle.publish(&submission.fence())?; + self.syncs.truncate(self.input_count); + } else { + install(self.syncs.drain(self.input_count..), &submission.fence()); + } Ok(submission) } pub(crate) fn finish(mut self, result: Result) -> Result { self.result = result; result } @@ -363,12 +386,19 @@ impl Outputs { impl Drop for Outputs { fn drop(&mut self) { if let Some(submission) = self.submission.take() { + if let Some(bundle) = &mut self.cumulative { + bundle.fail(&submission.fence()); + } submission.finish_enqueue(self.result); return; } if let Some(fallback) = self.fallback.take() { if let Err(error) = self.result { fallback.set_error(error); } - install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + if let Some(bundle) = &mut self.cumulative { + bundle.fail(&Fence::from_fence(&fallback)); + } else { + install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + } fallback.signal(); } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 0e29757a0c2cc..8c6891785f1c2 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -31,6 +31,7 @@ pub(crate) mod context; mod data_segment; mod dependency; mod sync_input; +pub(crate) mod sync_output; mod trace; mod event; mod feed; @@ -1499,7 +1500,8 @@ impl gpu::Gpu for Gpu { u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) + | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; if variant.few_primitives { params.features |= diff --git a/drivers/gpu/drm/asahi/g17/sync_input.c b/drivers/gpu/drm/asahi/g17/sync_input.c index e6b91988c3c28..19d5b8b92f934 100644 --- a/drivers/gpu/drm/asahi/g17/sync_input.c +++ b/drivers/gpu/drm/asahi/g17/sync_input.c @@ -3,31 +3,73 @@ #include #include #include "sync_input.h" +#include "sync_scope.h" + +static int collect_leaf(struct dma_fence *leaf, + struct asahi_g17_fence_leaf *out, size_t capacity, + size_t *count) +{ + if (*count == capacity) + return -ENOSPC; + out[*count].fence = dma_fence_get(leaf); + out[*count].status = dma_fence_get_status(leaf); + ++*count; + return 0; +} /* Called without the device mutex: status polling may run fence callbacks. */ int asahi_g17_collect_fences(struct dma_fence *head, struct asahi_g17_fence_leaf *out, size_t capacity) { - struct dma_fence_unwrap cursor; - struct dma_fence *leaf; - size_t count = 0; + struct dma_fence *pending[128]; + size_t count = 0, depth = 1; int error = 0; - dma_fence_unwrap_for_each(leaf, &cursor, head) { - if (dma_fence_is_container(leaf)) { - error = -EOPNOTSUPP; - break; + pending[0] = dma_fence_get(head); + while (depth && !error) { + struct dma_fence *node = pending[--depth]; + struct dma_fence_unwrap cursor; + struct dma_fence *leaf; + + if (asahi_g17_is_scope(node)) { + struct dma_fence *children[2]; + unsigned int n = asahi_g17_scope_children(node, children), i; + if (!n) { + /* Preserve a failed scope as a failed leaf, never drop its error. */ + if (dma_fence_get_status(node) < 0) + error = collect_leaf(node, out, capacity, &count); + } else if (depth + n > ARRAY_SIZE(pending)) { + for (i = 0; i < n; ++i) + dma_fence_put(children[i]); + error = -ENOSPC; + } else { + for (i = 0; i < n; ++i) + pending[depth++] = children[i]; + } + dma_fence_put(node); + continue; } - if (count == capacity) { - error = -ENOSPC; - break; + + dma_fence_unwrap_for_each(leaf, &cursor, node) { + if (asahi_g17_is_scope(leaf)) { + if (depth == ARRAY_SIZE(pending)) + error = -ENOSPC; + else + pending[depth++] = dma_fence_get(leaf); + } else if (dma_fence_is_container(leaf)) { + error = -EOPNOTSUPP; + } else { + error = collect_leaf(leaf, out, capacity, &count); + } + if (error) + break; } - out[count].fence = dma_fence_get(leaf); - out[count].status = dma_fence_get_status(leaf); - ++count; + /* A bounded early exit still owns the iterator's current chain. */ + dma_fence_put(cursor.chain); + dma_fence_put(node); } - /* A bounded early exit still owns the iterator's current chain. */ - dma_fence_put(cursor.chain); + while (depth) + dma_fence_put(pending[--depth]); if (error) { while (count) dma_fence_put(out[--count].fence); diff --git a/drivers/gpu/drm/asahi/g17/sync_output.c b/drivers/gpu/drm/asahi/g17/sync_output.c new file mode 100644 index 0000000000000..08b4a95f26c9b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include "sync_output.h" +#include "sync_scope.h" + +struct asahi_g17_signal_target { + struct drm_syncobj *syncobj; + struct dma_fence_chain *chain; + u64 point; +}; + +struct asahi_g17_signal_bundle { + size_t count; + size_t anchor; + bool published; + struct asahi_g17_signal_target targets[]; +}; + +void asahi_g17_signals_free(struct asahi_g17_signal_bundle *bundle) +{ + size_t i; + + if (!bundle) + return; + for (i = 0; i < bundle->count; ++i) { + struct asahi_g17_signal_target *target = &bundle->targets[i]; + if (target->chain) + dma_fence_chain_free(target->chain); + if (target->syncobj) + drm_syncobj_put(target->syncobj); + } + kfree(bundle); +} + +/* The Rust caller supplies its validated, immutable snapshot, never a user pointer. */ +struct asahi_g17_signal_bundle * +asahi_g17_signals_new(const struct asahi_g17_signal_spec *syncs, + size_t count, size_t anchor, int *error) +{ + struct asahi_g17_signal_bundle *bundle; + size_t i; + + *error = -EINVAL; + if (!count || anchor >= count || !syncs[anchor].point) + return NULL; + *error = -ENOMEM; + bundle = kzalloc(struct_size(bundle, targets, count), GFP_KERNEL); + if (!bundle) + return NULL; + bundle->count = count; + bundle->anchor = anchor; + for (i = 0; i < count; ++i) { + struct asahi_g17_signal_target *target = &bundle->targets[i]; + target->syncobj = syncs[i].syncobj; + if (target->syncobj) + drm_syncobj_get(target->syncobj); + if (!target->syncobj) { + *error = -ENOENT; + goto fail; + } + target->point = syncs[i].point; + if (syncs[i].timeline) { + target->chain = dma_fence_chain_alloc(); + if (!target->chain) + goto fail; + } + if (i != anchor && target->syncobj == bundle->targets[anchor].syncobj) { + *error = -EINVAL; + goto fail; + } + } + /* The anchor may follow the other outputs. Reject aliases in either order. */ + for (i = 0; i < count; ++i) { + if (i != anchor && bundle->targets[i].syncobj == bundle->targets[anchor].syncobj) { + *error = -EINVAL; + goto fail; + } + } + *error = 0; + return bundle; +fail: + asahi_g17_signals_free(bundle); + return NULL; +} + +static void asahi_g17_signal_install(struct asahi_g17_signal_target *target, + struct dma_fence *fence) +{ + if (target->chain) { + drm_syncobj_add_point(target->syncobj, target->chain, fence, target->point); + target->chain = NULL; + } else { + drm_syncobj_replace_fence(target->syncobj, fence); + } +} + +int asahi_g17_signals_publish(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *submitted) +{ + struct dma_fence *previous, *scope; + size_t i; + + if (bundle->published) + return -EINVAL; + previous = drm_syncobj_fence_get(bundle->targets[bundle->anchor].syncobj); + scope = asahi_g17_scope_create(previous, submitted, + bundle->targets[bundle->anchor].point); + dma_fence_put(previous); + if (!scope) + return -ENOMEM; + /* Only exported completion includes earlier work. GPU execution gets no new wait. */ + asahi_g17_signal_install(&bundle->targets[bundle->anchor], scope); + previous = drm_syncobj_fence_get(bundle->targets[bundle->anchor].syncobj); + asahi_g17_scope_bind(scope, previous); + dma_fence_put(previous); + for (i = 0; i < bundle->count; ++i) + if (i != bundle->anchor) + asahi_g17_signal_install(&bundle->targets[i], scope); + bundle->published = true; + dma_fence_put(scope); + return 0; +} + +/* Allocation or validation failed before enqueue: install an error promise without allocating. */ +void asahi_g17_signals_fail(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *failure) +{ + size_t i; + + if (bundle->published) + return; + for (i = 0; i < bundle->count; ++i) + asahi_g17_signal_install(&bundle->targets[i], failure); + bundle->published = true; +} diff --git a/drivers/gpu/drm/asahi/g17/sync_output.h b/drivers/gpu/drm/asahi/g17/sync_output.h new file mode 100644 index 0000000000000..a752d9ceaf4b0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_OUTPUT_H +#define ASAHI_G17_SYNC_OUTPUT_H +#include + +struct dma_fence; +struct drm_syncobj; +struct asahi_g17_signal_bundle; +struct asahi_g17_signal_spec { + struct drm_syncobj *syncobj; + u64 point; + u32 timeline; +}; + +struct asahi_g17_signal_bundle * +asahi_g17_signals_new(const struct asahi_g17_signal_spec *specs, + size_t count, size_t anchor, int *error); +void asahi_g17_signals_free(struct asahi_g17_signal_bundle *bundle); +int asahi_g17_signals_publish(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *submitted); +void asahi_g17_signals_fail(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *failure); +#endif diff --git a/drivers/gpu/drm/asahi/g17/sync_output.rs b/drivers/gpu/drm/asahi/g17/sync_output.rs new file mode 100644 index 0000000000000..50585befb3b71 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Atomic preparation of a queue completion scope and its exported signals. + +use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; + +// Wire values also documented in include/uapi/drm/asahi_drm.h. Keeping these +// local permits building this module against an older kernel's Rust bindings. +pub(crate) const SYNC_CUMULATIVE: u32 = 2; +pub(crate) const FEATURE_CUMULATIVE_SIGNALS: u64 = 1 << 17; + +unsafe extern "C" { + fn asahi_g17_signals_new(syncs: *const SignalSpec, count: usize, anchor: usize, + error: *mut i32) -> *mut core::ffi::c_void; + fn asahi_g17_signals_free(bundle: *mut core::ffi::c_void); + fn asahi_g17_signals_publish(bundle: *mut core::ffi::c_void, + current: *mut bindings::dma_fence) -> i32; + fn asahi_g17_signals_fail(bundle: *mut core::ffi::c_void, + failure: *mut bindings::dma_fence); +} + +#[repr(C)] +struct SignalSpec { + syncobj: *mut core::ffi::c_void, + point: u64, + timeline: u32, +} + +/// Owns syncobj references and preallocated chain nodes; never owns user pointers. +pub(crate) struct SignalBundle(*mut core::ffi::c_void); + +impl SignalBundle { + /// C retains references before the borrowed sync items can be dropped. + pub(crate) fn new(syncs: &[crate::file::SyncItem], anchor: usize) -> Result + { + let mut specs = KVec::with_capacity(syncs.len(), GFP_KERNEL)?; + for sync in syncs { + specs.push(SignalSpec { + syncobj: sync.syncobj.as_raw().cast(), point: sync.timeline_value, + timeline: u32::from(sync.timeline), + }, GFP_KERNEL)?; + } + let mut error = 0; + // SAFETY: All entries borrow live sync objects; C copies the entries and + // acquires its own references before returning. + let raw = unsafe { asahi_g17_signals_new( + specs.as_ptr(), specs.len(), anchor, &mut error) }; + if raw.is_null() { return Err(Error::from_errno(error)); } + Ok(Self(raw)) + } + + pub(crate) fn publish(&mut self, current: &Fence) -> Result { + // SAFETY: The bundle and current fence are owned for this call; C takes + // its own references on publication and reports failure before any store. + let error = unsafe { asahi_g17_signals_publish(self.0, current.raw()) }; + if error == 0 { Ok(()) } else { Err(Error::from_errno(error)) } + } + + pub(crate) fn fail(&mut self, failure: &Fence) { + // SAFETY: Live owned arguments; this only consumes uninstalled chain nodes. + unsafe { asahi_g17_signals_fail(self.0, failure.raw()) }; + } +} + +impl Drop for SignalBundle { + fn drop(&mut self) { + // SAFETY: Exactly one owner; published syncobjs retain their own fences. + unsafe { asahi_g17_signals_free(self.0) }; + } +} diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.c b/drivers/gpu/drm/asahi/g17/sync_scope.c new file mode 100644 index 0000000000000..55feb19a7f83d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_scope.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sync_scope.h" +#include "sync_input.h" + +struct asahi_g17_scope; +struct asahi_g17_scope_callback { + struct dma_fence_cb callback; + struct asahi_g17_scope *scope; +}; + +struct asahi_g17_scope { + struct dma_fence base; + atomic_t pending; + struct dma_fence *children[2]; + struct asahi_g17_scope_callback callbacks[2]; + struct irq_work work; + struct llist_node ready; + /* Identifies the exact anchor point this scope replaced. */ + u64 anchor_context; + u64 anchor_point; +}; + +static const struct dma_fence_ops scope_ops; +static DEFINE_PER_CPU(struct llist_head *, active_scopes); +#define SCOPE_ATOMIC_BATCH 64 + +bool asahi_g17_is_scope(struct dma_fence *fence) +{ + return fence && fence->ops == &scope_ops; +} + +static struct asahi_g17_scope *to_scope(struct dma_fence *fence) +{ + return container_of(fence, struct asahi_g17_scope, base); +} + +static const char *scope_driver_name(struct dma_fence *fence) +{ + return "asahi"; +} + +static const char *scope_timeline_name(struct dma_fence *fence) +{ + return "g17-queue-scope"; +} + +static bool scope_signaled(struct dma_fence *fence) +{ + struct asahi_g17_scope *scope = to_scope(fence); + if (atomic_read_acquire(&scope->pending)) + return false; + cmpxchg(&fence->error, 1, 0); + return true; +} + +static bool scope_enable_signaling(struct dma_fence *fence) +{ + /* Child callbacks are already armed; no recursive callback installation. */ + return !scope_signaled(fence); +} + +unsigned int asahi_g17_scope_children(struct dma_fence *fence, + struct dma_fence *children[2]) +{ + struct asahi_g17_scope *scope = to_scope(fence); + unsigned long flags; + unsigned int i, count = 0; + + spin_lock_irqsave(&fence->inline_lock, flags); + if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) + if (scope->children[i]) + children[count++] = dma_fence_get(scope->children[i]); + spin_unlock_irqrestore(&fence->inline_lock, flags); + return count; +} + +static void scope_release_completed(struct asahi_g17_scope *scope) +{ + struct dma_fence *children[2]; + unsigned long flags; + unsigned int i; + + /* Readers take references under this lock. Signalled scopes need no history. */ + spin_lock_irqsave(&scope->base.inline_lock, flags); + for (i = 0; i < ARRAY_SIZE(children); ++i) { + children[i] = scope->children[i]; + scope->children[i] = NULL; + } + spin_unlock_irqrestore(&scope->base.inline_lock, flags); + for (i = 0; i < ARRAY_SIZE(children); ++i) + dma_fence_put(children[i]); + dma_fence_put(&scope->base); +} + +static void scope_complete(struct asahi_g17_scope *scope) +{ + cmpxchg(&scope->base.error, 1, 0); + dma_fence_signal(&scope->base); + scope_release_completed(scope); +} + +static void scope_work(struct irq_work *work) +{ + struct asahi_g17_scope *scope = container_of(work, struct asahi_g17_scope, work); + cmpxchg(&scope->base.error, 1, 0); + asahi_g17_signal_fence(&scope->base); + scope_release_completed(scope); +} + +static void scope_defer_list(struct llist_node *nodes) +{ + struct llist_node *node, *next; + llist_for_each_safe(node, next, nodes) { + struct asahi_g17_scope *scope = + container_of(node, struct asahi_g17_scope, ready); + /* Transfer the ready-list reference to the irq_work callback. */ + irq_work_queue(&scope->work); + } +} + +void asahi_g17_signal_fence(struct dma_fence *fence) +{ + struct llist_head ready = LLIST_HEAD_INIT(ready); + struct llist_node *nodes, *node, *next; + unsigned long flags; + unsigned int completed = 0; + + /* Fence callbacks are atomic. Keep the local ready list pinned while + * signaling, then finish ready scopes after the source fence is unlocked. + * New ready scopes append to this list instead of recursing or sending IPIs. + */ + preempt_disable(); + if (__this_cpu_read(active_scopes)) { + dma_fence_signal(fence); + preempt_enable(); + return; + } + __this_cpu_write(active_scopes, &ready); + dma_fence_signal(fence); + for (;;) { + /* An interrupt must not append after the final empty test. */ + local_irq_save(flags); + nodes = llist_del_all(&ready); + if (!nodes) + __this_cpu_write(active_scopes, NULL); + local_irq_restore(flags); + if (!nodes) + break; + llist_for_each_safe(node, next, nodes) { + if (completed == SCOPE_ATOMIC_BATCH) { + struct llist_node *more; + local_irq_save(flags); + more = llist_del_all(&ready); + __this_cpu_write(active_scopes, NULL); + local_irq_restore(flags); + preempt_enable(); + scope_defer_list(node); + scope_defer_list(more); + return; + } + scope_complete(container_of(node, struct asahi_g17_scope, ready)); + ++completed; + } + } + preempt_enable(); +} + +static void scope_arrived(struct asahi_g17_scope *scope, int error) +{ + if (error < 0) + cmpxchg(&scope->base.error, 1, error); + if (atomic_dec_and_test(&scope->pending)) { + struct llist_head *ready; + dma_fence_get(&scope->base); + preempt_disable(); + ready = __this_cpu_read(active_scopes); + if (ready) + llist_add(&scope->ready, ready); + preempt_enable(); + /* Foreign fence signalers have no driver completion pass to drain us. */ + if (!ready) + irq_work_queue(&scope->work); + } +} + +static void scope_callback(struct dma_fence *child, struct dma_fence_cb *callback) +{ + struct asahi_g17_scope_callback *cb = + container_of(callback, typeof(*cb), callback); + scope_arrived(cb->scope, child->error); + dma_fence_put(&cb->scope->base); +} + +static void scope_release(struct dma_fence *fence) +{ + struct asahi_g17_scope *scope = to_scope(fence); + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) + dma_fence_put(scope->children[i]); + dma_fence_free(fence); +} + +static void scope_set_deadline(struct dma_fence *fence, ktime_t deadline) +{ + struct asahi_g17_fence_leaf *leaves; + int count, i; + + if (dma_fence_is_signaled(fence)) + return; + /* A deadline is an advisory hint. Keep its traversal bounded and iterative, + * including when a foreign timeline boundary contains earlier scopes. + */ + leaves = kmalloc_array(128, sizeof(*leaves), GFP_ATOMIC | __GFP_NOWARN); + if (!leaves) + return; + count = asahi_g17_collect_fences(fence, leaves, 128); + for (i = 0; i < count; ++i) { + if (!asahi_g17_is_scope(leaves[i].fence)) + dma_fence_set_deadline(leaves[i].fence, deadline); + dma_fence_put(leaves[i].fence); + } + kfree(leaves); +} + +static const struct dma_fence_ops scope_ops = { + .get_driver_name = scope_driver_name, + .get_timeline_name = scope_timeline_name, + .enable_signaling = scope_enable_signaling, + .signaled = scope_signaled, + .release = scope_release, + .set_deadline = scope_set_deadline, +}; + +void asahi_g17_scope_bind(struct dma_fence *fence, struct dma_fence *root) +{ + struct asahi_g17_scope *scope = to_scope(fence); + /* A concurrent raw syncobj reset is invalid for the private queue anchor, + * but must not turn a hostile userspace race into a kernel dereference. + */ + if (!root || dma_fence_chain_contained(root) != fence) + return; + /* Written once before the submit returns; an unmatched snapshot takes fallback. */ + WRITE_ONCE(scope->anchor_point, root->seqno); + WRITE_ONCE(scope->anchor_context, root->context); +} + +struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, + struct dma_fence *submitted, u64 point) +{ + struct dma_fence *contained = previous ? dma_fence_chain_contained(previous) : NULL; + struct dma_fence *parent = NULL; + struct asahi_g17_scope *scope, *prior = NULL; + u64 context; + unsigned int i; + + if (asahi_g17_is_scope(contained)) { + prior = to_scope(contained); + if (READ_ONCE(prior->anchor_context) != previous->context || + READ_ONCE(prior->anchor_point) != previous->seqno || + point <= contained->seqno) + prior = NULL; + } + if (prior) { + parent = dma_fence_get(contained); + context = contained->context; + } else { + /* Empty submissions or external edits may insert a foreign timeline point. + * Flatten that boundary once; the ordinary path retains just the prior scope. + */ + if (previous) { + parent = dma_fence_unwrap_merge(previous); + if (!parent) + return NULL; + } + context = dma_fence_context_alloc(1); + } + scope = kzalloc_obj(*scope); + if (!scope) { + dma_fence_put(parent); + return NULL; + } + dma_fence_init64(&scope->base, &scope_ops, NULL, context, point); + init_irq_work(&scope->work, scope_work); + scope->children[0] = parent; + scope->children[1] = dma_fence_get(submitted); + /* The constructor guard prevents completion while callbacks are being installed. */ + atomic_set(&scope->pending, 3); + scope->base.error = 1; + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) { + struct dma_fence *child = scope->children[i]; + if (!child) { + scope_arrived(scope, 0); + continue; + } + scope->callbacks[i].scope = scope; + dma_fence_get(&scope->base); + if (dma_fence_add_callback(child, &scope->callbacks[i].callback, scope_callback)) { + scope_arrived(scope, child->error); + dma_fence_put(&scope->base); + } + } + scope_arrived(scope, 0); + return &scope->base; +} diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.h b/drivers/gpu/drm/asahi/g17/sync_scope.h new file mode 100644 index 0000000000000..11c84d65f59f2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_scope.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_SCOPE_H +#define ASAHI_G17_SYNC_SCOPE_H +#include +struct dma_fence; +void asahi_g17_signal_fence(struct dma_fence *fence); + +struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, + struct dma_fence *submitted, u64 point); +void asahi_g17_scope_bind(struct dma_fence *scope, struct dma_fence *root); +bool asahi_g17_is_scope(struct dma_fence *fence); +/* Returns owned child references, or zero once the scope is signalled. */ +unsigned int asahi_g17_scope_children(struct dma_fence *fence, + struct dma_fence *children[2]); +#endif diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 66396f89cc333..836739ffbbebf 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -368,6 +368,13 @@ enum drm_asahi_feature { * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. */ DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, + + /** + * @DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS: SUBMIT accepts one + * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs + * include the earlier points of that timeline without adding GPU waits. + */ + DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 17, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -901,6 +908,16 @@ enum drm_asahi_sync_type { /** @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */ DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1, + + /** + * @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE: Output-only queue + * timeline anchor, requiring CUMULATIVE_SIGNALS and a nonzero point. + * At most one is allowed. All outputs receive completion of this submit + * and the earlier anchor points. + * Other outputs must not alias the anchor. No execution dependency is + * added. Calls updating this private anchor must be externally serialized. + */ + DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE = 2, }; /** diff --git a/rust/kernel/drm/syncobj.rs b/rust/kernel/drm/syncobj.rs index a022e08223588..72ba50c57286b 100644 --- a/rust/kernel/drm/syncobj.rs +++ b/rust/kernel/drm/syncobj.rs @@ -15,6 +15,14 @@ pub struct SyncObj { } impl SyncObj { + /// Borrows the raw sync object for synchronous calls into DRM helpers. + /// The caller must not release the owned reference or retain this pointer + /// beyond the lifetime of `self` without taking another reference. + #[inline(always)] + pub fn as_raw(&self) -> *mut bindings::drm_syncobj { + self.ptr + } + /// Looks up a sync object by its handle for a given `File`. pub fn lookup_handle( file: &drm::File, From bcdfdafd80cf93fadc79df688dd0109b9c4ae8d2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 03:45:04 -0400 Subject: [PATCH 34/64] drm/asahi: test G17 queue budgets with firmware-reserved identities Exercise dynamic render headroom and the full logical-queue budget with zero and nonzero firmware QID prefixes. Document that the shared watchdog counter orders reservations rather than concurrent firmware exposure. --- drivers/gpu/drm/asahi/g17/kick.rs | 35 +++++++++++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/port.rs | 3 +++ 2 files changed, 38 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index f3183bb005df6..a1f8f70f597d4 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -307,6 +307,41 @@ mod accounting_tests { assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); } + #[test] + fn dynamic_compute_growth_preserves_render_headroom_above_firmware_floor() { + for floor in [0, 4] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = 2; + let mut owner = 1; + while ids.compute_fits(queues) { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + owner += 1; + } + assert_eq!(ids.leased, QID_COUNT - floor - render_reserve(queues)); + assert_eq!(ids.reserve_for(owner, DataMaster::Compute, None, queues), Err(ENOSPC)); + assert!(ids.render_fits(queues + RENDER_HEADROOM)); + assert!(!ids.render_fits(queues + RENDER_HEADROOM + 1)); + } + } + + #[test] + fn full_logical_queue_budget_keeps_compute_floor_and_excludes_firmware_ids() { + for floor in [0, 4] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = queue_limit(floor) as usize; + for owner in 1..=COMPUTE_FLOOR as u64 { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + } + assert!(!ids.compute_fits(queues)); + assert!(ids.render_fits(queues)); + assert!(!ids.render_fits(queues + 1)); + } + } + #[test] fn initial_reservations_yield_but_published_ids_do_not() { let mut ids = table(); diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index d8f7f4ddff8a1..9acd9ff6e5910 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -371,6 +371,9 @@ impl Port { self.generation.store(generation, Ordering::Release); } + /// Unique reservation order shared by both engines. Concurrent publishers may expose + /// their work in a different order; queue serialization and visibility use other locks. + /// The watchdog uses this number only to choose bounded cross-VM timeout grace. pub(super) fn next_publication(&self) -> u64 { self.publications.fetch_add(1, Ordering::Relaxed) + 1 } From 971d4316d91cce97b92db72c4f29e5155b76d626 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 03:46:24 -0400 Subject: [PATCH 35/64] drm/asahi: size cached compute records for every native wait Dynamic QID admission permits 128 native inputs. A 0x1000-byte record slot holds only 18 waits at offset 0x900 with 0x60 bytes each, so the nineteenth could overwrite the following live slot. Size cached slots for the complete bounded list and validate each write against its own slot before deriving an allocation offset. Keep announcement slots unchanged. Host execution of the production offset helper and regression test bodies covers all 128 waits in all 32 retained slots; the old-size control fails. --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 62 ++++++++++++++++++---- 1 file changed, 51 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 6ca63595154de..02ae0bbbb9634 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -30,6 +30,12 @@ const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; +const NATIVE_WAIT_OFFSET: usize = 0x900; +const NATIVE_WAIT_STRIDE: usize = 0x60; +// A submission can name every physical compute channel. Keep the complete +// wait list inside its own cached slot, including the last retained slot. +const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET + + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE).next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; @@ -193,8 +199,8 @@ impl Graph { let cached_records = objects.allocate( alloc, Family::QueueConfig, - RECORD_SLOTS * RECORD_STRIDE, - RECORD_SLOTS * RECORD_STRIDE, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + RECORD_SLOTS * CACHED_RECORD_STRIDE, prot, cpu, )?; @@ -1196,6 +1202,32 @@ fn release_pending(released: bool, quarantined: bool, active: usize, retiring: b mod admission_tests { use super::*; + #[test] + fn complete_native_wait_list_fits_every_cached_slot() { + let waits = crate::g17::kick::COMPUTE_CHANNELS; + for slot in 0..RECORD_SLOTS { + for index in 0..waits { + let start = record_start(slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).unwrap(); + assert!(start >= slot * CACHED_RECORD_STRIDE); + assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); + } + } + } + + #[test] + fn record_window_cannot_overwrite_the_next_slot() { + // The nineteenth wait used to cross the old slot boundary silently. + let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; + assert_eq!(record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), Err(EINVAL)); + assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); + assert_eq!(record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), + Ok(CACHED_RECORD_STRIDE - 1)); + } + #[test] fn retained_ring_pressure_waits_without_discarding_work() { let mut active = Fifo::new(); @@ -1760,6 +1792,15 @@ impl Queue { } } +/// Validate the individual slot before deriving an offset into its allocation. +/// Allocation-wide checks cannot detect a write into the following live slot. +fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Result { + if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { + return Err(EINVAL); + } + slot.checked_mul(stride).and_then(|start| start.checked_add(offset)).ok_or(EINVAL) +} + impl Graph { fn record( &mut self, @@ -1769,17 +1810,15 @@ impl Graph { value: T, ) -> Result { let (object, start) = if let Some(slot) = slot { - if slot >= RECORD_SLOTS { - return Err(EINVAL); - } - let object: &mut KernelObject = if offset == ANNOUNCE { - &mut self.records + let (object, stride): (&mut KernelObject, usize) = if offset == ANNOUNCE { + (&mut self.records, RECORD_STRIDE) } else { - &mut self.cached_records + (&mut self.cached_records, CACHED_RECORD_STRIDE) }; - (object, slot * RECORD_STRIDE + offset) + (object, record_start(slot, offset, window, stride)?) } else { - (&mut *self.graph, 0x6800 + offset) + let start = record_start(0, offset, window, GRAPH_SIZE - 0x6800)?; + (&mut *self.graph, 0x6800 + start) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -2031,7 +2070,8 @@ impl Queue { for (index, wait) in native.iter().enumerate() { let record = wait.bytes(crate::g17::fw::stamp_value(kick_timestamp.get() as u32), packet.order.sequence); - let address = self.graph.record(record_slot, 0x900 + index * 0x60, 0x60, record)?; + let address = self.graph.record(record_slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, record)?; wait_vas.push(address, GFP_KERNEL)?; } let prepared_kick = self.kick.prepare_entry(&kick_args)?; From 9bf3039873a1e241991b5999aceff5610c660ade Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 04:00:14 -0400 Subject: [PATCH 36/64] drm/asahi: index pending compute completion identities before reading payloads The shared completion ring retains old records for active QIDs. Checking each one against every pending descriptor reads unrelated payload words and repeats a linear search, especially when a future pending command keeps the full scan open. Build a bounded per-QID kick index in the existing observation batch. Reject old kicks from the control word, then retain the exact descriptor, validity and timestamp checks. Keep all scan requests, full-ring fallbacks and retirement semantics unchanged. Add collision, descriptor-reuse, stale-payload, clear and partial-publication tests. --- .../drm/asahi/g17/queue/compute/retirement.rs | 156 ++++++++++++++++-- 1 file changed, 140 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index b268c1c40de00..46df3598362b7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -166,12 +166,22 @@ struct Request { descriptor: u64, kick: u64, observed: Option, + next: u16, +} + +const NO_REQUEST: u16 = u16::MAX; +const _: () = assert!(QIDS * IN_FLIGHT < NO_REQUEST as usize); + +fn request_bucket(qid: usize, kick: u64) -> usize { + qid * IN_FLIGHT + kick as usize % IN_FLIGHT } /// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; /// staging and folding never allocate under the device mutex. pub(crate) struct Batch { requests: KVVec, + /// Per-QID kick buckets; collisions retain the complete identity check. + heads: KVVec, slots: [(usize, usize); QIDS], open: Option, /// Requests with no observation yet; the scan stops when this reaches zero. @@ -180,8 +190,13 @@ pub(crate) struct Batch { impl Batch { pub(crate) fn new() -> Result { + let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; + for _ in 0..QIDS * IN_FLIGHT { + heads.push_within_capacity(NO_REQUEST).map_err(|_| ENOSPC)?; + } Ok(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, + heads, slots: [(0, 0); QIDS], open: None, unobserved: 0, @@ -209,45 +224,74 @@ impl Batch { return Err(ENOSPC); } let start = if slot.1 == 0 { + let begin = ticket.qid as usize * IN_FLIGHT; + self.heads.get_mut(begin..begin + IN_FLIGHT).ok_or(EIO)?.fill(NO_REQUEST); self.requests.len() } else { slot.0 }; + let bucket = request_bucket(ticket.qid as usize, ticket.kick); + let next = *self.heads.get(bucket).ok_or(EIO)?; + let index = u16::try_from(self.requests.len()).map_err(|_| ENOSPC)?; self.requests .push_within_capacity(Request { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, + next, }) .map_err(|_| ENOSPC)?; + *self.heads.get_mut(bucket).ok_or(EIO)? = index; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); self.unobserved += 1; Ok(()) } - /// Capture the control word once, then read only a requested identity's payload. - /// Records can be overwritten during a scan: incomplete or older identities are skipped. + /// Capture the control word once. Most slots belong to an older kick of a + /// selected QID; reject those before reading any additional shared words. pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - let control = record.read(0)?; + self.fold_reader(|word| record.read(word)) + } + + fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { + let control = read(0)?; let qid = ((control >> 9) & 0x7f) as usize; - let (start, count) = self.slots[qid]; + let (_, count) = self.slots[qid]; if count == 0 { return Ok(()); } - let descriptor = record.read(6)? & !1; - if descriptor == 0 || record.read(7)? == 0 { - return Ok(()); - } let kick = (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; - let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; - let Some(request) = requests - .iter_mut() - .find(|r| r.descriptor == descriptor && r.kick == kick) - else { - return Ok(()); - }; - let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; + let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; + let mut descriptor = None; + let mut found = None; + for _ in 0..count { + if index == NO_REQUEST { + break; + } + let request = self.requests.get(usize::from(index)).ok_or(EIO)?; + if request.kick == kick { + let value = if let Some(value) = descriptor { + value + } else { + let value = read(6)? & !1; + if value == 0 || read(7)? == 0 { + return Ok(()); + } + descriptor = Some(value); + value + }; + if request.descriptor == value { + // Chains run newest first. Retain the original first-match + // rule even if a caller stages the same identity twice. + found = Some(usize::from(index)); + } + } + index = request.next; + } + let Some(index) = found else { return Ok(()); }; + let request = self.requests.get_mut(index).ok_or(EIO)?; + let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; if timestamps[0] == 0 || timestamps[1] == 0 || request @@ -415,3 +459,83 @@ mod tests { assert_eq!(result, Err(ENODATA)); } } + + +#[kunit_tests(asahi_g17_completion_index)] +mod index_tests { + use super::*; + + fn ticket(qid: u8, kick: u64, descriptor: u64) -> Ticket { + Ticket { owner: 1, submission: 1, context_id: 1, qid, descriptor, + work_node: 0x2000, kick, ordinal: 1, item_producer: 4, item_count: 64, epoch: 1 } + } + fn record(ticket: &Ticket, timestamps: [u64; 2]) -> [u64; 8] { + let mut words = [0; 8]; + words[0] = ((ticket.kick & !0xff) << 8) | ((ticket.kick & 0xff) << 48) + | (u64::from(ticket.qid) << 9); + words[2] = timestamps[0]; words[5] = timestamps[1]; + words[6] = ticket.descriptor | 1; words[7] = 1; + words + } + #[test] + fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000)).unwrap(); + let words = record(&ticket(7, 17, 0x1000), [10, 20]); + batch.fold_reader(|word| { + assert_eq!(word, 0); + Ok(words[word]) + }).unwrap(); + assert!(!batch.complete()); + } + #[test] + fn collisions_and_reused_descriptors_keep_full_kick_identity() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 49, 0x1000); + batch.request(&first).unwrap(); batch.request(&second).unwrap(); + let words = record(&first, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(!batch.complete()); + let words = record(&second, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.take(&second).unwrap().unwrap().timestamps, [30, 40]); + } + #[test] + fn same_kick_still_checks_descriptor_and_zero_duration_is_retained() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 33, 0x2000); + batch.request(&first).unwrap(); batch.request(&second).unwrap(); + let words = record(&first, [20, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); + assert!(batch.take(&second).unwrap().is_none()); + } + #[test] + fn duplicate_identity_keeps_the_first_staged_request() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t).unwrap(); batch.request(¤t).unwrap(); + let words = record(¤t, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.unobserved, 1); + } + #[test] + fn clear_restarts_buckets_and_later_publication_stays_unobserved() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000)).unwrap(); batch.clear(); + let current = ticket(7, 49, 0x1000); + batch.request(¤t).unwrap(); + let stale = record(&ticket(7, 33, 0x1000), [10, 20]); + batch.fold_reader(|word| Ok(stale[word])).unwrap(); + assert!(!batch.complete()); + assert!(batch.take(&ticket(7, 50, 0x2000)).unwrap().is_none()); + let words = record(¤t, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + } +} From 26446182da3aa8d98ce799478a025117b021eaba Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 04:16:00 -0400 Subject: [PATCH 37/64] drm/asahi: retain a deep working set of validated GPU page ranges Cache the full page footprint already checked by the VM, preserve permission-subset and mapping-epoch checks, and merge only compatible intervals. Keep the transient parser bound at 48 requests while retaining 128 intervals in the persistent cache so a depth-64 stream does not repeatedly flush every proof. Measured normal runs reduce compute depth-64 submit CPU from about 10.06 to 7.46 us and render depth-32 from 12.29 to 8.16 us. Wall throughput remains limited elsewhere. Range, permission, epoch, overflow and rotating-working-set regression tests pass. --- drivers/gpu/drm/asahi/g17/validation.rs | 174 ++++++++++++++++++++++-- 1 file changed, 164 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index b2a6cf6e8bfe3..b0f2861382ad7 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -8,19 +8,41 @@ use core::{cell::RefCell, ops::Range}; use kernel::prelude::*; const MAX_RANGES: usize = 48; +// The persistent proof must retain a deep queue's rotating command buffers. +// Keep the transient parser request set small to bound its stack frame. +const MAX_CACHED_RANGES: usize = 128; type Request = (u64, u64, u8); +/// Match the page footprint already checked by `Vm::covers_range_locked`. +fn pages((address, size, access): Request) -> Option { + if size == 0 || access == 0 || access & !3 != 0 { + return None; + } + let mask = mmu::UAT_PGSZ as u64 - 1; + let end = address.checked_add(size)?; + let end = end.checked_add(mask)? & !mask; + let start = address & !mask; + Some((start, end.checked_sub(start)?, access)) +} + +fn contains((start, size, access): Request, (other, length, needed): Request) -> bool { + access & needed == needed + && other >= start + && other.checked_add(length).is_some_and(|end| + start.checked_add(size).is_some_and(|limit| end <= limit)) +} + #[derive(Clone, Copy)] -struct Requests { - ranges: [Request; MAX_RANGES], +struct Requests { + ranges: [Request; CAPACITY], count: usize, overflow: bool, } -impl Requests { +impl Requests { const fn new() -> Self { Self { - ranges: [(0, 0, 0); MAX_RANGES], + ranges: [(0, 0, 0); CAPACITY], count: 0, overflow: false, } @@ -30,24 +52,55 @@ impl Requests { &self.ranges[..self.count] } + fn covers(&self, request: Request) -> bool { + self.as_slice().iter().any(|&range| contains(range, request)) + } + fn insert(&mut self, request: Request) { - if self.as_slice().contains(&request) { + let Some(mut request) = pages(request) else { + self.overflow = true; + return; + }; + if self.covers(request) { return; } - if self.count == MAX_RANGES { + // Merge only equal permissions, so an adjacent writable page never + // grants write access to a read-only page. Restart after each merge + // because it can join a range visited earlier. + let mut index = 0; + while index < self.count { + let range = self.ranges[index]; + let end = request.0 + request.1; + let other_end = range.0 + range.1; + if contains(request, range) + || (request.2 == range.2 && request.0 <= other_end && range.0 <= end) + { + if request.2 == range.2 { + let start = request.0.min(range.0); + request = (start, end.max(other_end) - start, request.2); + } + self.count -= 1; + self.ranges[index] = self.ranges[self.count]; + index = 0; + } else { + index += 1; + } + } + if self.count == CAPACITY { self.overflow = true; return; } self.ranges[self.count] = request; self.count += 1; } + } -/// One logical queue's bounded cache of exact address, size and permission tuples. +/// One logical queue's bounded cache of validated page ranges and permissions. /// Mapping mutations make the epoch unavailable until their final page-table write. pub(crate) struct Proof { epoch: Option, - requests: Requests, + requests: Requests, } impl Proof { @@ -64,7 +117,7 @@ impl Proof { && requests .as_slice() .iter() - .all(|request| self.requests.as_slice().contains(request)) + .all(|&request| self.requests.covers(request)) } fn record(&mut self, epoch: Option, requests: Requests) { @@ -79,7 +132,10 @@ impl Proof { } // An incomplete union cannot be reused; retain the latest complete set. self.epoch = if requests.overflow { None } else { epoch }; - self.requests = requests; + self.requests = Requests::new(); + for &request in requests.as_slice() { + self.requests.insert(request); + } } /// The VM-job guard must already pin the mappings for this command. Keep the @@ -144,3 +200,101 @@ impl AddressSpace for Space<'_> { } } } + + +#[kunit_tests(asahi_g17_validation_pages)] +mod tests { + use super::*; + const PAGE: u64 = mmu::UAT_PGSZ as u64; + + fn requests(range: Request) -> Requests { + let mut result = Requests::new(); + result.insert(range); + result + } + + #[test] + fn different_offsets_share_the_already_validated_page() { + let mut proof = Proof::new(); + proof.record(Some(7), requests((PAGE + 4, 4, 3))); + assert!(proof.covers(Some(7), &requests((PAGE + 0x100, 8, 1)))); + assert!(proof.covers(Some(7), &requests((2 * PAGE - 8, 8, 2)))); + assert!(!proof.covers(Some(7), &requests((2 * PAGE - 8, 16, 1)))); + assert!(!proof.covers(Some(8), &requests((PAGE + 4, 4, 1)))); + assert!(!proof.covers(None, &requests((PAGE + 4, 4, 1)))); + } + + #[test] + fn adjacent_equal_permissions_merge_but_holes_do_not() { + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((3 * PAGE, 1, 1)); + assert_eq!(ranges.count, 2); + assert!(!ranges.covers((2 * PAGE, PAGE, 1))); + ranges.insert((2 * PAGE, 1, 1)); + assert_eq!(ranges.count, 1); + assert_eq!(ranges.as_slice(), &[(PAGE, 3 * PAGE, 1)]); + } + + #[test] + fn adjacent_permissions_never_grant_access_to_another_page() { + let mut proof = Proof::new(); + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((2 * PAGE, 1, 3)); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((PAGE, 1, 2)))); + assert!(proof.covers(Some(1), &requests((2 * PAGE, 1, 2)))); + assert!(!proof.covers(Some(1), &requests((PAGE, 2 * PAGE, 3)))); + } + + #[test] + fn stronger_coverage_removes_only_fully_covered_ranges() { + let mut ranges = requests((PAGE, PAGE, 1)); + ranges.insert((2 * PAGE, PAGE, 2)); + ranges.insert((PAGE, 2 * PAGE, 3)); + assert_eq!(ranges.as_slice(), &[(PAGE, 2 * PAGE, 3)]); + assert!(!ranges.covers((0, PAGE, 1))); + } + + #[test] + fn persistent_cache_retains_a_deep_rotating_working_set() { + let mut proof = Proof::new(); + for index in 0..64 { + proof.record(Some(7), requests((index * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 64); + for index in 0..64 { + assert!(proof.covers(Some(7), &requests((index * 2 * PAGE + 16, 8, 1)))); + } + assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); + } + + #[test] + fn persistent_overflow_keeps_only_a_new_complete_proof() { + let mut proof = Proof::new(); + for index in 0..=MAX_CACHED_RANGES { + proof.record(Some(7), requests((index as u64 * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 1); + assert!(!proof.covers(Some(7), &requests((0, 8, 1)))); + assert!(proof.covers(Some(7), + &requests((MAX_CACHED_RANGES as u64 * 2 * PAGE, 8, 1)))); + } + + #[test] + fn invalid_rounding_and_full_cache_disable_reuse() { + for range in [(0, 0, 1), (u64::MAX - 1, 1, 1), (0, 1, 0), (0, 1, 4)] { + assert!(pages(range).is_none()); + let mut proof = Proof::new(); + proof.record(Some(1), requests(range)); + assert!(!proof.covers(Some(1), &requests((0, 1, 1)))); + } + let mut ranges = Requests::new(); + for index in 0..=MAX_RANGES { + ranges.insert(((2 * index) as u64 * PAGE, PAGE, 1)); + } + assert!(ranges.overflow); + let mut proof = Proof::new(); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((0, PAGE, 1)))); + } +} From b6248acf900c1cf30deac52791c2c50d10055d7b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 05:14:09 -0400 Subject: [PATCH 38/64] drm/asahi: avoid locking pending private completion fences Frontier pruning examines independently signalled driver completion fences. Check their signalled flag before taking the generic status lock; retain the locked terminal path for errors and leave external input fence handling unchanged. Seven Neo depth-64 repetitions reduce median submit API elapsed time from 7.52 to 5.79 us. Frontier scan CPU samples fall from 1.83% to 0.25% of the machine; throughput remains about 19.5 us per command. The full benchmark matrix, candidate-bound short stress, software-fence/error gates and 20-source native fan-in pass. --- drivers/gpu/drm/asahi/g17/job.rs | 12 ++++++++++++ drivers/gpu/drm/asahi/g17/queue/mod.rs | 8 ++++---- drivers/gpu/drm/asahi/g17/sync_scope.c | 12 ++++++++++++ drivers/gpu/drm/asahi/g17/sync_scope.h | 2 ++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 572cfd5d0f382..27e54613965cd 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -385,6 +385,18 @@ pub(crate) struct Fences { pub(crate) publication: Option>, } +impl Fences { + /// Scheduler::enqueue always supplies our explicit completion fence here. + /// It has no polling callback, so pending frontier checks need no lock. + pub(crate) fn status(&self) -> i32 { + unsafe extern "C" { + fn asahi_g17_completion_fence_status(fence: *mut bindings::dma_fence) -> i32; + } + // SAFETY: completed owns the live private fence throughout this read. + unsafe { asahi_g17_completion_fence_status(self.completed.raw()) } + } +} + impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index c959f8bc8c4b8..0cc64b8b94460 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -155,7 +155,7 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; self.frontiers[index].retain(|(sequence, fences)| { - let status = job::fence_status(&fences.completed); + let status = fences.status(); if status == 0 { return true; } @@ -203,11 +203,11 @@ impl Queue { && !destinations.has_outputs() && self.frontiers[0].iter().all(|(sequence, fences)| !Order::contains(order.wait_through[0], *sequence) - || job::fence_status(&fences.completed) > 0) + || fences.status() > 0) && self.frontiers[1].iter().all(|(sequence, fences)| !Order::contains(order.wait_through[1], *sequence) || fences.ready.raw() != fences.completed.raw() - || job::fence_status(&fences.completed) > 0) + || fences.status() > 0) && self.failures.iter().enumerate().all(|(index, failure)| failure.as_ref().is_none_or(|(sequence, _)| !Order::contains(order.wait_through[index], *sequence))) @@ -293,7 +293,7 @@ impl Queue { // Successful completion includes host timestamp publication. Its // ready fence adds no ordering and would prevent shallow inline // publication. Failed and pending producers still need checking. - if job::fence_status(&fences.completed) > 0 { + if fences.status() > 0 { continue; } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.c b/drivers/gpu/drm/asahi/g17/sync_scope.c index 55feb19a7f83d..a3153f3122b94 100644 --- a/drivers/gpu/drm/asahi/g17/sync_scope.c +++ b/drivers/gpu/drm/asahi/g17/sync_scope.c @@ -11,6 +11,18 @@ #include "sync_scope.h" #include "sync_input.h" +int asahi_g17_completion_fence_status(struct dma_fence *fence) +{ + /* These private fences have no signaled callback. A pending flag is + * sufficient to keep the frontier member; a concurrent signal can only + * defer pruning until the next submission. Read terminal errors through + * the normal locked API after observing the signalled flag. + */ + if (!dma_fence_test_signaled_flag(fence)) + return 0; + return dma_fence_get_status(fence); +} + struct asahi_g17_scope; struct asahi_g17_scope_callback { struct dma_fence_cb callback; diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.h b/drivers/gpu/drm/asahi/g17/sync_scope.h index 11c84d65f59f2..e88b78aacdd7b 100644 --- a/drivers/gpu/drm/asahi/g17/sync_scope.h +++ b/drivers/gpu/drm/asahi/g17/sync_scope.h @@ -4,6 +4,8 @@ #include struct dma_fence; void asahi_g17_signal_fence(struct dma_fence *fence); +/* Only for explicitly signalled driver completion fences, with no poll op. */ +int asahi_g17_completion_fence_status(struct dma_fence *fence); struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, struct dma_fence *submitted, u64 point); From b08c81722d896eee8da9771e99e38e6986a2a806 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 23:19:10 -0400 Subject: [PATCH 39/64] drm/asahi: give G17 firmware restarts time to drain long work The restart handshake waited 500 one-millisecond ticks for each of its two phases. The firmware completes a restart only after the work it interrupted has drained, and with several clients running long ray-traversal jobs that takes seconds: six concurrent ray-query stress clients timed the second phase out in state 2, and the host then declared the whole device lost, killing every client including the desktop. Wait up to ten seconds per phase, the longest execution allowance a job gets, and log which phase timed out and the state the firmware was in, as well as a restart storm stop. With this the same load completes every restart and only the clients whose work was blamed or timed out see a lost device. --- drivers/gpu/drm/asahi/g17/recovery.rs | 10 +++++++++- drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 89f8d33bdb80c..8aac25ab80d68 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -24,7 +24,11 @@ const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; const MAX_GUILTY_VMS: usize = 8; -const RECOVERY_POLLS: usize = 500; +/// One-millisecond ticks per handshake phase. The firmware finishes a restart only after the +/// work it interrupted has drained; with several clients running long ray-traversal jobs that +/// takes seconds, and giving up loses every client's device. Ten seconds matches the longest +/// execution allowance (`timeout::POLL_MS` plus renewals). +const RECOVERY_POLLS: usize = 10_000; /// Per-QID classifications from the complete firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. @@ -192,6 +196,8 @@ pub(super) fn service( } } if !ready { + pr_warn!("G17: recovery gen {} did not reach state 1 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // Recheck ownership after the off-CPU waits, before any handshake mutation. @@ -217,6 +223,8 @@ pub(super) fn service( host.wait_tick(); } if !recovered { + pr_warn!("G17: recovery gen {} did not reach state 3 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index cdd8cc6d5dea5..c1a2ebe232fcb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -266,7 +266,10 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { - if self.recovery.restart_limit(super::now_ns()) { return Err(ETIMEDOUT); } + if self.recovery.restart_limit(super::now_ns()) { + pr_warn!("G17: restart storm limit reached\n"); + return Err(ETIMEDOUT); + } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, From f1db16a83586e4bec596a71e12e0d8b87dd3b16c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 24 Oct 2026 23:23:44 -0400 Subject: [PATCH 40/64] drm/asahi: fail commands a GPU restart interrupted instead of completing them The firmware recovery table lists every command that was in flight when the firmware stopped: one record per command, with its queue and its completion stamp. The driver only folded the guilty and cancelled bits of those records into per-queue masks and ignored the rest. An innocent command that was interrupted part way could then still be retired by its stamps and completion record, and report success with partial output. Six concurrent ray query stress cases showed exactly that: after recoveries that cancelled nothing, three clients completed without error and with wrong results. Read the table as the packed list it is, up to its first invalid record, and mark every listed command interrupted before the firmware resumes. An interrupted command never reports success. Its failure always reaches the client, without poisoning the VM unless a recovery found it guilty; ECANCELED is no longer silent for it, and its output is not handed to consumers as valid. It is replayed only when it provably never started: a compute command with a zero-duration record, or a render group with neither its tiler start nor its fragment end written. A render group interrupted after its tiler started fails at once rather than being replayed, since blending or storage writes could apply twice. The recovery log line reports how many listed commands matched by stamp and how many fell back to the oldest command of their queue. --- drivers/gpu/drm/asahi/g17/completion.rs | 56 ++++++++-- drivers/gpu/drm/asahi/g17/initdata.rs | 23 ++-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 21 ++++ drivers/gpu/drm/asahi/g17/queue/render.rs | 103 +++++++++++++++++- .../drm/asahi/g17/queue/render/retirement.rs | 5 + drivers/gpu/drm/asahi/g17/recovery.rs | 71 +++++++++++- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 11 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 20 +++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 16 +++ 10 files changed, 301 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index d692a98acea8c..11b4c323d4668 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -270,6 +270,9 @@ pub(crate) struct Completion { /// A recovery blamed this command. Unlike `class`, which a later /// classification can change, this never clears. blamed: AtomicBool, + /// A firmware restart listed this command as in flight. Until a replay + /// republishes it from the start, it never reports success. + interrupted: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } @@ -327,6 +330,7 @@ impl Completion { replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), blamed: AtomicBool::new(false), + interrupted: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -379,6 +383,11 @@ impl Completion { } /// Called under the device mutex as a successful retirement is queued. pub(super) fn note_source_retired(&self) { + if self.interrupted() { + // complete() fails it; consumers must not take its output as valid. + self.note_input_error(ECANCELED); + return; + } match self.check_source_inputs() { Ok(()) => self.output.output().member.note_retired(), Err(error) => { self.note_input_error(error); } @@ -429,6 +438,9 @@ impl Completion { .is_ok(); if accepted { self.replay_timeout.store(true, Ordering::Release); + // Callers replay only work proven not to have started; the new + // publication runs from the beginning and reports its own result. + self.interrupted.store(false, Ordering::Release); } accepted } @@ -517,6 +529,35 @@ impl Completion { self.blamed.load(Ordering::Acquire) } + /// A firmware restart listed this command in its recovery table: it was in + /// flight, so it may have run partly. Its stamps and completion record may + /// still look like success. Unless a replay republishes it from the start, + /// it fails and the failure is always reported to the client. + pub(crate) fn note_interrupted(&self) { + self.interrupted.store(true, Ordering::Release); + } + + pub(crate) fn interrupted(&self) -> bool { + self.interrupted.load(Ordering::Acquire) + } + + /// Whether this failure makes the VM unusable. Spared work, and interrupted + /// work no recovery found guilty, fail without poisoning it. + pub(crate) fn poisons(&self, error: Error) -> bool { + let interrupted = self.interrupted(); + (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) + } + + /// ECANCELED stays silent only for work that was never interrupted (closed + /// or cancelled by its owner); every other failure reaches the client. + fn report(&self, error: Error) { + if self.poisons(error) { + self.status.record(error); + } else if error != ECANCELED || self.interrupted() { + self.status.report_failure(error); + } + } + /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -589,6 +630,9 @@ impl Completion { if self.signalled.swap(true, Ordering::AcqRel) { return; } + if result.is_ok() && self.interrupted() { + result = Err(ECANCELED); + } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { // Compute reports its dispatch envelope in the first pair, a // render its fragment pass in the second. @@ -600,13 +644,7 @@ impl Completion { } if let Err(error) = result { // Spared work cancelled after another VM's recovery is not this client's device loss. - if self.spared() { - if error != ECANCELED { - self.status.report_failure(error); - } - } else if error != ECANCELED { - self.status.record(error); - } + self.report(error); } self.release_healthy(); self.signal_terminal(result); @@ -640,8 +678,8 @@ impl Completion { } if self.spared() { self.status.report_failure(error); - } else if error != ECANCELED { - self.status.record(error); + } else { + self.report(error); } self.signal_terminal(Err(error)); self.output.finish(Err(error)); diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 8f7551b1763c9..0b92aa3cdd1d4 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -1564,17 +1564,22 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entries(&self) -> Result { + fn entry(&self, index: usize) -> Result<(u32, u32)> { + if index >= super::recovery::RECOVERY_RECORDS { + return Err(EINVAL); + } let cluster = self.object(self.cluster)?; - let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; + let offset = self.status_b + + abi::PrimaryStatusB::RECOVERY_INFO + + index * core::mem::size_of::(); fence(Ordering::Acquire); - let mut entries = super::recovery::Entries::default(); - for index in 0..256 { - let offset = base + index * core::mem::size_of::(); - let flags = cluster.word(offset)?.load(Ordering::Relaxed); - entries.include(flags); - } - Ok(entries) + let flags = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? + .load(Ordering::Relaxed); + let stamp = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, stamp))? + .load(Ordering::Relaxed); + Ok((flags, stamp)) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 02ae0bbbb9634..16d6351678bb7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -718,6 +718,27 @@ impl Queue { visit(active.packet.completion.status()); } } + /// A recovery record names this queue and the completion stamp of the + /// command it interrupted, `kick << 8` like the queue's stamp. Without a + /// stamp match the oldest published command is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self.active.iter().find(|active| { + active + .ticket + .is_some_and(|ticket| (ticket.kick as u32) & 0xff_ffff == sequence) + }) { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.ticket.is_some()) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { for active in self.active.iter() { active diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 3e65ea3b9a367..1d562da115b60 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1088,6 +1088,28 @@ impl Pair { .front() .map(|active| active.packet.completion.spared()) } + /// A recovery record names one of this pair's queues and the completion + /// stamp of the group it interrupted, `(ordinal + 1) << 8` like the pair's + /// stamps. Without a stamp match the oldest published group is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self + .active + .iter() + .filter(|active| active.outer_published) + .find(|active| (active.ticket.ordinal.wrapping_add(1) as u32) & 0xff_ffff == sequence) + { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.outer_published) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { for active in self.active.iter() { visit(active.packet.completion.status()); @@ -1271,6 +1293,9 @@ impl Pair { if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { return Ok(false); } + if let Some(progressed) = self.settle_interrupted_front(host, defer, grace)? { + return Ok(progressed); + } use crate::g17::replay::{self, Decision, Group}; let qids = self.qids(); let memory = &self.memory; @@ -1295,10 +1320,11 @@ impl Pair { let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; // A torn sample is no proof; a later poll or the timeout looks again. + // Interrupted work is run again only if it never started. Ok(matches!( retirement::abandoned(&active.ticket, snapshot, stamps, state, false), Ok(true) - )) + ) && (!active.packet.completion.interrupted() || snapshot.untouched())) })() .unwrap_or_else(|error: Error| { failure = Some(error); @@ -1391,6 +1417,81 @@ impl Pair { Ok(true) } + /// A front group a restart interrupted after its tiler started, which firmware + /// then retired without writing its fragment end, may have run partly. It is + /// not replayed (blending or storage writes could apply twice) and fails now, + /// reported to its owner, instead of waiting out the render timeout. Returns + /// None when the front is not such a group. + fn settle_interrupted_front( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, + ) -> Result> { + use retirement::Memory; + let qids = self.qids(); + let Some(front) = self.active.front() else { + return Ok(None); + }; + if !front.outer_published || front.failed || !front.packet.completion.interrupted() { + return Ok(None); + } + let ordinal = front.ticket.ordinal; + let ticket = front.ticket; + let mut observer = Observer { + memory: &self.memory, + qids, + host: &mut *host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if snapshot.untouched() + || !matches!( + retirement::abandoned(&ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + return Ok(None); + } + let now = crate::g17::now_ns(); + let front = self.active.front_mut().ok_or(EIO)?; + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(Some(false)); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(Some(false)); + } + } + if !front.flist_completed { + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + pr_warn!( + "G17: render slot {}: group {} was interrupted by a GPU recovery after it started; failing it\n", + self.slot, + ordinal + ); + let packet = self.active.front().ok_or(EIO)?.packet.clone(); + defer(Deferred::Retired(packet.completion.clone(), Err(ECANCELED)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + Ok(Some(true)) + } + /// The pending replay cannot proceed (closed owner, cancellation or a /// publication refusal). Its successors cannot run without it and fail with /// it. ECANCELED leaves the VM usable; any other error is the ordinary diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 24aeed6bcd0ec..00e4997ef3661 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -122,6 +122,11 @@ impl Snapshot { fn executed(self) -> bool { self.pass[0] != 0 && self.pass[1] != 0 } + + /// Neither the tiler start nor the fragment end was written: nothing ran. + pub(crate) fn untouched(self) -> bool { + self.pass == [0, 0] + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 8aac25ab80d68..cfadf4bb31767 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -30,18 +30,28 @@ const MAX_GUILTY_VMS: usize = 8; /// execution allowance (`timeout::POLL_MS` plus renewals). const RECOVERY_POLLS: usize = 10_000; -/// Per-QID classifications from the complete firmware recovery table. +/// Records in the firmware recovery table. +pub(super) const RECOVERY_RECORDS: usize = 256; + +/// Per-QID classifications from the firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. #[derive(Default)] pub(super) struct Entries { guilty: u128, cancelled: u128, + /// Valid records, each naming one command that was in flight at the restart. + pub(super) listed: u32, + /// Listed commands found by their exact stamp, and by the oldest-work fallback. + pub(super) exact: u32, + pub(super) fallback: u32, } impl Entries { - pub(super) fn include(&mut self, flags: u32) { + /// Returns false at the first invalid record: the table is a packed list and + /// records after it are left over from an earlier restart. + pub(super) fn include(&mut self, flags: u32) -> bool { if flags & 1 == 0 { - return; + return false; } let qid = (flags >> 4) & 0x7f; let bit = 1u128 << qid; @@ -51,9 +61,22 @@ impl Entries { if flags & u32::from(CANCELLED) != 0 { self.cancelled |= bit; } + self.listed += 1; + true } } +/// How a listed recovery record was matched to retained work. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Interrupted { + /// No retained command of the queue. + None, + /// The command whose completion stamp equals the record's. + Exact, + /// No stamp matched; the queue's oldest unretired command was marked instead. + Oldest, +} + /// How an error completion affects its VM's sticky status and other jobs. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Class { @@ -75,6 +98,10 @@ pub(super) trait Queues { fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); /// Classify every in-flight completion; return QIDs of render slots with spared work. fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; + /// Mark the command a recovery record lists as interrupted. It may have + /// partly executed, so it must never report success; see + /// `Completion::note_interrupted`. + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; @@ -89,7 +116,9 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - fn entries(&self) -> Result; + /// One recovery record's flags and completion stamp, read while firmware + /// is halted in state 3. + fn entry(&self, index: usize) -> Result<(u32, u32)>; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -228,14 +257,26 @@ pub(super) fn service( return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let entries = host.entries().unwrap_or_default(); + // Every listed command was in flight when firmware stopped. Firmware may + // still retire it with stamps and records that look like success, so it is + // marked before firmware resumes and never reports success. + let entries = read_entries(host); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; + let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); + if listed != 0 { + pr_warn!( + "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", + listed, + exact, + fallback + ); + } if let Some(qid) = request.blamed { pr_warn!( "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", @@ -255,6 +296,26 @@ pub(super) fn service( Ok(ServiceResult { scan_error, target_error }) } +/// Walks the packed recovery table up to its first invalid record and marks +/// every listed command interrupted. A failed read keeps what was read so far. +fn read_entries(host: &mut H) -> Entries { + let mut entries = Entries::default(); + for index in 0..RECOVERY_RECORDS { + let Ok((flags, stamp)) = host.entry(index) else { + break; + }; + if !entries.include(flags) { + break; + } + match host.note_interrupted(((flags >> 4) & 0x7f) as u8, stamp) { + Interrupted::Exact => entries.exact += 1, + Interrupted::Oldest => entries.fallback += 1, + Interrupted::None => {} + } + } + entries +} + /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index c30e45e22579a..47675c9638395 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -74,6 +74,17 @@ mod tests { for _ in 0..1024 { assert!(storm.note(u64::MAX)); } } #[test] + fn recovery_table_ends_at_its_first_invalid_record() { + use crate::g17::recovery::Entries; + let mut entries = Entries::default(); + assert!(entries.include(1 | (12 << 4))); + assert!(entries.include(1 | 2 | (17 << 4))); + assert!(!entries.include(0)); + assert_eq!(entries.listed, 2); + assert_eq!(entries.guilty, 1u128 << 17); + assert_eq!(entries.cancelled, 0); + } + #[test] fn replay_dependency_checks_never_wait_for_event_service() { use crate::g17::dependency::replay_status; assert_eq!(replay_status(0), Err(EBUSY)); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index b07423bd22115..f50e8e350e936 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1275,7 +1275,7 @@ impl DeferredBatch { // it. Expose terminal VM status now, before off-lock fence callbacks. // Recovery-spared and cancelled commands never poison their VM. if let Deferred::FailedOwned(completion, error) = &value { - if *error != ECANCELED && !completion.spared() { + if completion.poisons(*error) { completion.status().record(*error); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index c1a2ebe232fcb..e13ec3e35e13c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -131,8 +131,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entries(&self) -> Result { - self.firmware.init.entries() + fn entry(&self, index: usize) -> Result<(u32, u32)> { + self.firmware.init.entry(index) } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -203,6 +203,22 @@ impl recovery::Queues for Service<'_> { } self.firmware.queues.classify_render(class) } + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> recovery::Interrupted { + let render = self.firmware.queues.note_render_interrupted(qid, stamp); + if render != recovery::Interrupted::None { + return render; + } + self.firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.qid() == qid) + .map_or(recovery::Interrupted::None, |queue| { + queue.note_interrupted(stamp) + }) + } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index f4f45a98005c3..4aaad40eee765 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -830,6 +830,22 @@ impl Registry { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_spared()) } + /// Marks the interrupted group of the pair that owns `qid`. + pub(in crate::g17) fn note_render_interrupted( + &self, + qid: u8, + stamp: u32, + ) -> crate::g17::recovery::Interrupted { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .map_or(crate::g17::recovery::Interrupted::None, |pair| { + pair.note_interrupted(stamp) + }) + } pub(in crate::g17) fn visit_render_owners( &self, qid: u8, From 30f85560fd28ce05151c16eab7a58433d125a81e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 00:00:33 -0400 Subject: [PATCH 41/64] drm/asahi: stop the G17 device on a watchdog storm only without progress Eight job watchdog expirations within 30 seconds stopped the device, even while other work kept retiring. Under overload (six concurrent ray-query stress clients) that happened within a minute: each timeout already failed its own client, yet the ninth took every other client and the desktop with it. Count retired commands and start a new storm window whenever work has retired since the previous expiration. A device that stops retiring anything still stops after eight; an overloaded one only loses the clients whose work timed out. --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 1 + drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + drivers/gpu/drm/asahi/g17/recovery.rs | 5 +++- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 26 +++++++++++++++++++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 16d6351678bb7..c0916b639ee97 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -968,6 +968,7 @@ impl Queue { } let front = self.active.front().ok_or(EIO)?; if let Some(ticket) = front.ticket { + crate::g17::recovery::note_retired(); crate::g17::trace::complete(crate::g17::trace::COMPUTE, ticket.qid, ticket.ordinal, &front.packet, [timestamps[0], timestamps[1]]); } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 1d562da115b60..5c201ee6ec965 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1260,6 +1260,7 @@ impl Pair { front.qos_completed = true; front.flist_completed = true; } + crate::g17::recovery::note_retired(); crate::g17::trace::complete(crate::g17::trace::RENDER, qids[1], ordinal as u32, &front.packet, [timestamps[0], timestamps[3]]); defer(Deferred::Retired( diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index cfadf4bb31767..84a01d975c36d 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -10,6 +10,7 @@ mod attribution; mod slot; mod safety; +pub(crate) use safety::note_retired; pub(crate) use attribution::Sources; use attribution::Attribution; @@ -152,7 +153,9 @@ impl State { restarts: safety::Storm::new(), } } - pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note(now) } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { + self.timeouts.note_unless_progress(now, safety::retired()) + } pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index 47675c9638395..b97fc26914329 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -2,17 +2,41 @@ //! Fail closed on repeated watchdog/restart traffic, even across client loss. +use core::sync::atomic::{AtomicU64, Ordering}; + const WINDOW_NS: u64 = 30_000_000_000; const LIMIT: u8 = 8; +/// Commands retired successfully. A watchdog storm means the GPU stopped making progress; +/// timeouts of overloaded clients while other work keeps retiring are not one. +static RETIRED: AtomicU64 = AtomicU64::new(0); + +/// Called for every command that retires without error. +pub(crate) fn note_retired() { + RETIRED.fetch_add(1, Ordering::Relaxed); +} + +pub(super) fn retired() -> u64 { + RETIRED.load(Ordering::Relaxed) +} + pub(super) struct Storm { start: Option, count: u8, stopped: bool, + progress: u64, } impl Storm { pub(super) const fn new() -> Self { - Self { start: None, count: 0, stopped: false } + Self { start: None, count: 0, stopped: false, progress: 0 } + } + /// Like [`Self::note`], but work retired since the previous event starts a new window. + pub(super) fn note_unless_progress(&mut self, now: u64, progress: u64) -> bool { + if !self.stopped && progress != self.progress { + self.progress = progress; + self.start = None; + } + self.note(now) } pub(super) fn note(&mut self, now: u64) -> bool { if self.stopped { return true; } From 2c0abe2d090a9701b13b22d5ac56fb24d87cfe12 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 03:50:55 -0400 Subject: [PATCH 42/64] drm/asahi: scan only the G17 compute completion records written since the last pass The completion scan walked the shared compute completion ring from its cursor until it had found a record for every pending command. A command still running has no record yet, so every pass with work in flight read all 2048 slots: about 3 us on the completion path before any fence was signalled, and the cursor never advanced while a queue stayed busy. Records are appended in order, so remember each slot's control word as of the last pass over it; the first slot that still holds it is where the firmware writes next, and the pass ends there. Stage the pending identities of every queue rather than only those the batch retires, so each new record is folded once and later batches find it latched. A record of a command published after staging, or one whose payload is still being written, ends the pass and is read again by the next one. A command whose completion stamp already covers it has its record in the ring; if a pass ends without it, the whole ring is read once and the cursor moves after the newest record. Same-boot A/B against the previous scan (us per submission, medians of two rounds, app pinned to cpu0 / cpu5): compute.light d2 33.7 -> 31.0 / 43.0 -> 33.8, d64 26.6 -> 20.4 / 14.2 -> 14.3, render.light d32 38.6 -> 34.8 / 30.4 -> 29.8; d1, render d1/d2, queue hops and 16-process compute unchanged. Event worker CPU time at compute d64 drops by a quarter. --- drivers/gpu/drm/asahi/g17/initdata.rs | 84 ++++++-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 27 ++- .../drm/asahi/g17/queue/compute/retirement.rs | 203 ++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 36 +++- 4 files changed, 291 insertions(+), 59 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 0b92aa3cdd1d4..7e0c36ece43ad 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -38,6 +38,26 @@ pub(crate) struct ComputeCompletion<'a> { offset: usize, } +/// Records in the compute completion ring that the host scans. +pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { + let slots = abi::COMPLETION_RING_SIZE / 0x40; + if slots < 0x800 { slots } else { 0x800 } +}; + +/// Timestamp bits of a completion record's start and end words. +const COMPLETION_TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; + +/// What one completion-ring slot gave a scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CompletionFold { + /// No new observation of a pending request; the scan continues. + Passed, + /// Every request now has an observation. + Done, + /// A pending request's record whose payload the firmware is still writing. + Incomplete, +} + impl ComputeCompletion<'_> { pub(crate) fn read(&self, word: usize) -> Result { if word >= 8 { @@ -711,28 +731,66 @@ impl InitData { /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. - /// Visits ring slots from `start`, wrapping once, until `record` reports - /// that nothing more is wanted. Returns the slot after the last one visited - /// when the scan stopped early, or `start` after a complete wrap. - pub(super) fn scan_compute_completions( + /// Folds the compute completion records the firmware appended since `start`, in ring + /// order. `seen` holds every slot's control word as of the previous pass over it: the + /// first slot still holding it is where the firmware writes next, and the pass ends there + /// instead of reading the rest of the ring. A pass also ends once `record` has found every + /// request, or at a requested record whose payload is still being written, which the next + /// pass reads again. Returns the slot the next pass starts at. + pub(super) fn fold_compute_completions( &self, start: usize, - mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + seen: &mut [u64], + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + ) -> Result { + let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } + fence(Ordering::Acquire); + let completion = |slot: usize| ComputeCompletion { + object, + offset: slot * 0x40, + }; + super::queue::compute::retirement::walk_completions( + start, + seen, + |slot| completion(slot).read(0), + |slot| record(completion(slot)), + ) + } + + /// Folds every slot of the compute completion ring and records its control word in + /// `seen`. For a pass that stopped before a record the firmware must already have + /// written. Returns the slot after the newest record (latest end timestamp), where the + /// firmware writes next. + pub(super) fn resync_compute_completions( + &self, + seen: &mut [u64], + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { let object = self.object(self.completion_rings[0])?; - let slots = (abi::COMPLETION_RING_SIZE / 0x40).min(0x800); - let start = start % slots; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } fence(Ordering::Acquire); - for step in 0..slots { - let slot = (start + step) % slots; - if record(ComputeCompletion { + let mut newest = None; + for slot in 0..COMPUTE_COMPLETION_SLOTS { + let completion = ComputeCompletion { object, offset: slot * 0x40, - })? { - return Ok((slot + 1) % slots); + }; + let control = completion.read(0)?; + seen[slot] = control; + if control != 0 { + let end = completion.read(5)? & COMPLETION_TIME_MASK; + if newest.is_none_or(|(_, latest)| end > latest) { + newest = Some((slot, end)); + } } + record(completion)?; } - Ok(start) + Ok(newest.map_or(0, |(slot, _)| (slot + 1) % COMPUTE_COMPLETION_SLOTS)) } fn object(&self, index: usize) -> Result<&KernelObject> { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index c0916b639ee97..c8e00e2b563c2 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -786,33 +786,32 @@ impl Queue { || self.retire_pending || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) } - /// Stage every unlatched exact identity before the device's single shared-ring scan. - pub(crate) fn request_observations( - &self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + /// Stage every unlatched exact identity before the device's single shared-ring scan, + /// whichever queues the scan's batch retires. + pub(crate) fn request_observations(&self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(false); } + batch.stage_frontier(self.qid(), self.kick.timestamp().get())?; let mut requested = false; + let mut stamp = None; for active in self.active.iter() { let Some(ticket) = active.ticket else { break; }; if active.observed.is_none() { - batch.request(&ticket)?; + let stamp = match stamp { + Some(stamp) => stamp, + None => *stamp.insert(self.graph.stamp()?), + }; + batch.request(&ticket, retirement::stamp_covers(stamp, ticket.kick))?; requested = true; } } Ok(requested) } - pub(crate) fn latch_observations( - &mut self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + pub(crate) fn latch_observations(&mut self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(()); } for active in self.active.iter_mut() { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 46df3598362b7..c2b49e7b436d7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,7 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. -use crate::g17::initdata::ComputeCompletion; +use crate::g17::initdata::{CompletionFold, ComputeCompletion}; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -161,11 +161,46 @@ pub(crate) fn quarantined_retired( && submitted == pending) } +/// Walks a completion ring of `seen.len()` slots from `start` over the records written since +/// the previous walk: `seen` holds each slot's control word as of the last walk that passed it, +/// so the first slot still holding it is where the firmware writes next. `record` folds one +/// slot; the walk also ends once it reports every request found, or at a requested record +/// still being written, which the next walk reads again. Returns where the next walk starts. +pub(crate) fn walk_completions( + start: usize, + seen: &mut [u64], + mut control: impl FnMut(usize) -> Result, + mut record: impl FnMut(usize) -> Result, +) -> Result { + let slots = seen.len(); + if slots == 0 { + return Err(EINVAL); + } + let start = start % slots; + for step in 0..slots { + let slot = (start + step) % slots; + let word = control(slot)?; + if word == seen[slot] { + return Ok(slot); + } + match record(slot)? { + CompletionFold::Incomplete => return Ok(slot), + CompletionFold::Passed => seen[slot] = word, + CompletionFold::Done => { + seen[slot] = word; + return Ok((slot + 1) % slots); + } + } + } + Ok(start) +} + #[derive(Clone, Copy)] struct Request { descriptor: u64, kick: u64, observed: Option, + covered: bool, next: u16, } @@ -186,8 +221,18 @@ pub(crate) struct Batch { open: Option, /// Requests with no observation yet; the scan stops when this reaches zero. unobserved: usize, + /// Unobserved requests whose queue's completion stamp already covered them when staged. + /// The firmware stores the stamp after the record, so each has a record in the ring. + covered: usize, + /// Per QID, the next kick its queue had not published when the batch was staged + /// ([`NO_FRONTIER`] for a QID without a staged queue). A record at or past it belongs + /// to a command published during the scan, which a later batch stages. + frontiers: [u64; QIDS], } +const NO_FRONTIER: u64 = u64::MAX; +const KICK_MASK: u64 = (1 << 40) - 1; + impl Batch { pub(crate) fn new() -> Result { let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; @@ -200,6 +245,8 @@ impl Batch { slots: [(0, 0); QIDS], open: None, unobserved: 0, + covered: 0, + frontiers: [NO_FRONTIER; QIDS], }) } pub(crate) fn clear(&mut self) { @@ -207,12 +254,29 @@ impl Batch { self.slots.fill((0, 0)); self.open = None; self.unobserved = 0; + self.covered = 0; + self.frontiers.fill(NO_FRONTIER); + } + /// Records the first kick `qid`'s queue had not yet published when this batch was staged. + pub(crate) fn stage_frontier(&mut self, qid: u8, next: u64) -> Result { + *self.frontiers.get_mut(usize::from(qid)).ok_or(EINVAL)? = next & KICK_MASK; + Ok(()) + } + /// A record of `qid` with `kick` names a command published after this batch was staged. + fn published_after_staging(&self, qid: usize, kick: u64) -> bool { + let frontier = self.frontiers[qid]; + frontier != NO_FRONTIER && kick.wrapping_sub(frontier) & KICK_MASK < 1 << 39 } /// True once every requested identity has an observation. pub(crate) fn complete(&self) -> bool { self.unobserved == 0 } - pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { + /// Some request whose record must already be in the ring is still unobserved. + pub(crate) fn covered_pending(&self) -> bool { + self.covered != 0 + } + /// Stages `ticket`; `covered` when its queue's completion stamp already covers its kick. + pub(crate) fn request(&mut self, ticket: &Ticket, covered: bool) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { return Err(EINVAL); } @@ -238,6 +302,7 @@ impl Batch { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, + covered, next, }) .map_err(|_| ENOSPC)?; @@ -245,23 +310,37 @@ impl Batch { *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); self.unobserved += 1; + self.covered += usize::from(covered); Ok(()) } /// Capture the control word once. Most slots belong to an older kick of a /// selected QID; reject those before reading any additional shared words. - pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - self.fold_reader(|word| record.read(word)) + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + if !self.fold_reader(|word| record.read(word))? { + return Ok(CompletionFold::Incomplete); + } + Ok(self.folded()) + } + + /// Like [`Self::fold`], for a scan that reads every slot regardless of the outcome. + pub(crate) fn fold_any(&mut self, record: ComputeCompletion<'_>) -> Result { + self.fold_reader(|word| record.read(word)).map(|_| ()) } - fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { + fn folded(&self) -> CompletionFold { + if self.complete() { CompletionFold::Done } else { CompletionFold::Passed } + } + + /// Ok(false) when a pending request's record is still being written. + fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { let control = read(0)?; let qid = ((control >> 9) & 0x7f) as usize; let (_, count) = self.slots[qid]; - if count == 0 { - return Ok(()); - } let kick = (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; + if count == 0 { + return Ok(!self.published_after_staging(qid, kick)); + } let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; let mut descriptor = None; let mut found = None; @@ -276,7 +355,7 @@ impl Batch { } else { let value = read(6)? & !1; if value == 0 || read(7)? == 0 { - return Ok(()); + return Ok(request.observed.is_some()); } descriptor = Some(value); value @@ -289,23 +368,28 @@ impl Batch { } index = request.next; } - let Some(index) = found else { return Ok(()); }; + let Some(index) = found else { + return Ok(!self.published_after_staging(qid, kick)); + }; let request = self.requests.get_mut(index).ok_or(EIO)?; let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; - if timestamps[0] == 0 - || timestamps[1] == 0 - || request - .observed - .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + if timestamps[0] == 0 || timestamps[1] == 0 { + return Ok(request.observed.is_some()); + } + if request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) { - return Ok(()); + return Ok(true); } if request.observed.is_none() { self.unobserved -= 1; + self.covered -= usize::from(request.covered); } request.observed = Some(Observation { timestamps }); - Ok(()) + Ok(true) } + /// A command published after this batch staged its queue's requests was not part of the /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { @@ -480,7 +564,7 @@ mod index_tests { #[test] fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 33, 0x1000)).unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); let words = record(&ticket(7, 17, 0x1000), [10, 20]); batch.fold_reader(|word| { assert_eq!(word, 0); @@ -493,7 +577,7 @@ mod index_tests { let mut batch = Batch::new().unwrap(); let first = ticket(7, 33, 0x1000); let second = ticket(7, 49, 0x1000); - batch.request(&first).unwrap(); batch.request(&second).unwrap(); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); let words = record(&first, [10, 20]); batch.fold_reader(|word| Ok(words[word])).unwrap(); assert!(!batch.complete()); @@ -508,7 +592,7 @@ mod index_tests { let mut batch = Batch::new().unwrap(); let first = ticket(7, 33, 0x1000); let second = ticket(7, 33, 0x2000); - batch.request(&first).unwrap(); batch.request(&second).unwrap(); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); let words = record(&first, [20, 20]); batch.fold_reader(|word| Ok(words[word])).unwrap(); assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); @@ -518,7 +602,7 @@ mod index_tests { fn duplicate_identity_keeps_the_first_staged_request() { let mut batch = Batch::new().unwrap(); let current = ticket(7, 33, 0x1000); - batch.request(¤t).unwrap(); batch.request(¤t).unwrap(); + batch.request(¤t, false).unwrap(); batch.request(¤t, false).unwrap(); let words = record(¤t, [10, 20]); batch.fold_reader(|word| Ok(words[word])).unwrap(); assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); @@ -527,9 +611,9 @@ mod index_tests { #[test] fn clear_restarts_buckets_and_later_publication_stays_unobserved() { let mut batch = Batch::new().unwrap(); - batch.request(&ticket(7, 33, 0x1000)).unwrap(); batch.clear(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); batch.clear(); let current = ticket(7, 49, 0x1000); - batch.request(¤t).unwrap(); + batch.request(¤t, false).unwrap(); let stale = record(&ticket(7, 33, 0x1000), [10, 20]); batch.fold_reader(|word| Ok(stale[word])).unwrap(); assert!(!batch.complete()); @@ -538,4 +622,77 @@ mod index_tests { batch.fold_reader(|word| Ok(words[word])).unwrap(); assert!(batch.complete()); } + #[test] + fn a_requested_record_still_being_written_is_read_again() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, true).unwrap(); + let mut words = record(¤t, [10, 20]); + words[7] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + words[7] = 1; + words[5] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.covered_pending()); + let words = record(¤t, [10, 20]); + assert!(batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.complete() && !batch.covered_pending()); + } + #[test] + fn a_command_published_during_the_scan_is_left_for_the_next_batch() { + let mut batch = Batch::new().unwrap(); + batch.stage_frontier(7, 50).unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + batch.stage_frontier(9, 3).unwrap(); + let late = record(&ticket(7, 50, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(late[word])).unwrap()); + let idle = record(&ticket(9, 3, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(idle[word])).unwrap()); + let earlier = record(&ticket(9, 2, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(earlier[word])).unwrap()); + let wrapped = record(&ticket(7, 0xff_ffff_fff0, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(wrapped[word])).unwrap()); + } + fn walk(start: usize, seen: &mut [u64], ring: &[u64], outcome: impl Fn(usize) -> CompletionFold) + -> usize + { + walk_completions(start, seen, |slot| Ok(ring[slot]), |slot| Ok(outcome(slot))).unwrap() + } + #[test] + fn a_walk_reads_only_the_records_written_since_the_previous_one() { + let mut seen = [0u64; 8]; + let mut ring = [0u64; 8]; + assert_eq!(walk(0, &mut seen, &ring, |_| CompletionFold::Passed), 0); + ring[0] = 10; ring[1] = 11; ring[2] = 12; + let read = core::cell::Cell::new(0); + assert_eq!(walk(0, &mut seen, &ring, |_| { read.set(read.get() + 1); CompletionFold::Passed }), 3); + assert_eq!(read.get(), 3); + ring[3] = 13; + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + // A full lap later the firmware overwrote slots 4..8 and 0..2 with newer records. + for (slot, word) in [(4, 14), (5, 15), (6, 16), (7, 17), (0, 18), (1, 19)] { ring[slot] = word; } + assert_eq!(walk(4, &mut seen, &ring, |_| CompletionFold::Passed), 2); + } + #[test] + fn a_walk_stops_at_a_partial_record_and_after_the_last_request() { + let mut seen = [0u64; 8]; + let ring = [10, 11, 12, 13, 0, 0, 0, 0]; + assert_eq!(walk(0, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Incomplete } + else { CompletionFold::Passed }), 2); + assert_eq!(seen[..3], [10, 11, 0]); + assert_eq!(walk(2, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Done } + else { CompletionFold::Passed }), 3); + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + } + #[test] + fn an_unrequested_or_stale_record_is_passed() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + let mut stale = record(&ticket(7, 33, 0x1000), [10, 20]); + stale[7] = 0; + assert!(batch.fold_reader(|word| Ok(stale[word])).unwrap()); + let other = record(&ticket(9, 33, 0x1000), [10, 20]); + assert!(batch.fold_reader(|word| Ok(other[word])).unwrap()); + assert!(!batch.complete() && !batch.covered_pending()); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index f50e8e350e936..7f89a25ae6d35 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -178,6 +178,8 @@ pub(super) struct Registry { pub(super) observations: compute::retirement::Batch, /// Completion-ring slot where the next compute scan starts. completion_cursor: usize, + /// Control word of each completion-ring slot as of the last scan that read it. + completion_seen: KVVec, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -227,6 +229,14 @@ impl Registry { replay_outer_wait: None, observations: compute::retirement::Batch::new()?, completion_cursor: 0, + completion_seen: { + let slots = super::initdata::COMPUTE_COMPLETION_SLOTS; + let mut seen = KVVec::with_capacity(slots, GFP_KERNEL)?; + for _ in 0..slots { + seen.push_within_capacity(0).map_err(|_| ENOMEM)?; + } + seen + }, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) @@ -1620,25 +1630,33 @@ impl super::Firmware { let registry = &mut self.queues; registry.observations.clear(); let mut requested = false; + // Every queue's pending identities are staged, not only the ones this batch + // retires: the scan passes each new record once, and a queue retired by a later + // batch finds its observation already latched. for queue in registry .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) { - requested |= queue.request_observations(&mut registry.observations, masks)?; + requested |= queue.request_observations(&mut registry.observations)?; } - // Completion records are appended in order: resume after the last - // record a previous scan needed and stop once every request is found. + // Completion records are appended in order: fold only the records written since + // the previous scan. A request whose stamp already covers it has its record in the + // ring; if the scan ended before finding one, read the whole ring once. if requested { let observations = &mut registry.observations; - registry.completion_cursor = self.init.scan_compute_completions( + let seen = &mut registry.completion_seen; + registry.completion_cursor = self.init.fold_compute_completions( registry.completion_cursor, - |record| { - observations.fold(record)?; - Ok(observations.complete()) - }, + seen, + |record| observations.fold(record), )?; + if observations.covered_pending() { + registry.completion_cursor = self + .init + .resync_compute_completions(seen, |record| observations.fold_any(record))?; + } } let port = &self.port; let retired = deferred.pending(); @@ -1650,7 +1668,7 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) { let busy = !queue.channel_free(); - queue.latch_observations(&mut registry.observations, masks)?; + queue.latch_observations(&mut registry.observations)?; queue.retire( masks, generation, From 39bdd5b8e2d584fff0d342a71a8945e64c8a9673 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:58 -0400 Subject: [PATCH 43/64] drm/asahi: release pruned G17 frontier entries after the ioctl publishes Pruning a logical queue's frontier at the start of every enqueue dropped the last references of the commands it found finished. That tears their completions down, including the deferred timestamp cleanup, before the new command is prepared, so its doorbell waited for the teardown. Keep the finished entries until the ioctl has enqueued everything and release them then. Without room for them they are released at once, as before. Prune results, failure tracking and dependencies are unchanged. Same-boot phase timers, render.light d1 pinned to cpu5: the prune step before publication 857 -> 269 ns per submit; the teardown now follows the doorbell. --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 0cc64b8b94460..16891524e25a0 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -95,6 +95,9 @@ pub(crate) struct Queue { proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], + /// Finished frontier entries pruned during this ioctl. Their last references + /// can tear a whole completion down; that waits until the ioctl has published. + finished: KVec, failures: [Option<(u64, Fence)>; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. @@ -133,6 +136,7 @@ impl Queue { proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), + finished: KVec::new(), failures: [None, None], _slot: slot, }) @@ -154,11 +158,20 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; + // Without room the entries are released here, as they always could be. + let mut finished = if self.finished.reserve(self.frontiers[index].len(), GFP_KERNEL).is_ok() { + Some(&mut self.finished) + } else { + None + }; self.frontiers[index].retain(|(sequence, fences)| { let status = fences.status(); if status == 0 { return true; } + if let Some(finished) = finished.as_mut() { + let _ = finished.push_within_capacity(fences.clone()); + } // One earliest failed ordinal covers every later prefix without // retaining its packet, mappings, or every failed fence. if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { @@ -461,6 +474,8 @@ impl crate::queue::Queue for Queue { } Ok(()) })(); + // Everything this ioctl publishes is visible; release what it pruned. + self.finished.clear(); if let Err(error) = result { self.context.status().record_if_device_loss(error); if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { From b41ecb56c0775a43b747adaa37661b353651dbff Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:59 -0400 Subject: [PATCH 44/64] drm/asahi: publish both G17 render QoS owners in one pass Each render publication published its fragment and tiling QoS owners one after the other. Every publish read back the row it had just stored byte by byte and read-modify-wrote the submission counters, all in the uncached QoS table, and the second publish read words in the cache lines the first had just written, so its reads waited for those stores to drain. Publish a command's owners together: check every owner first, read each table word once before the first store, then store the identities, one fence, and the counters. The firmware-visible values and their order per owner are those of one publish per owner; a refused publication now changes nothing instead of being cancelled. Compute keeps its single owner and gets the same read-before-store pass. Same-boot phase timers, pinned to cpu5: render QoS publication 1041 + 265 -> 262 ns per submit; compute ioctl entry to doorbell 9.05 -> 8.44 us. --- drivers/gpu/drm/asahi/g17/port.rs | 11 ++ drivers/gpu/drm/asahi/g17/qos.rs | 143 ++++++++++++------ drivers/gpu/drm/asahi/g17/queue/render.rs | 17 ++- drivers/gpu/drm/asahi/g17/runtime/render.rs | 8 +- .../drm/asahi/g17/runtime/render/direct.rs | 8 +- 5 files changed, 127 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 9acd9ff6e5910..45ddae9719363 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -333,6 +333,17 @@ impl Port { self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock) } + /// Publishes the two owners of a render command, fragment then tiling. + pub(super) fn qos_publish_pair( + &self, + owners: [qos::Owner; 2], + scheduler: u64, + policy: Policy, + ) -> Result<[qos::Publication; 2]> { + let view = self.qos_view()?; + self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock) + } + pub(super) fn qos_cancel(&self, publication: qos::Publication) -> Result { let view = self.qos_view()?; self.qos.lock().cancel(&view, publication) diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index 4f6913ad5ee95..b36656b0174a7 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -53,6 +53,14 @@ impl<'a> View<'a> { fn row(&self, qid: usize) -> [u8; 8] { core::array::from_fn(|index| self.rows[qid][index].load(Ordering::Relaxed)) } + /// The complete row in one access. The table is uncached: a publication reads + /// each row once, before it stores anything. + fn row_word(&self, qid: usize) -> [u8; 8] { + let row = core::ptr::from_ref(&self.rows[qid]).cast::(); + // SAFETY: Rows are eight bytes at eight-byte offsets of the eight-byte + // aligned table (checked in from_raw), so the row is one aligned word. + unsafe { (*row).load(Ordering::Relaxed) }.to_le_bytes() + } } #[derive(Clone, Copy, Debug)] @@ -108,66 +116,111 @@ impl Accounting { policy: Policy, clock: impl FnOnce() -> (u64, u64), ) -> Result { - let (qid, qos, dm) = owner.indices()?; + let [publication] = self.publish_owners(view, [owner], scheduler, policy, clock)?; + Ok(publication) + } + + /// Publishes the owners of one command's queues, in order, with the same + /// firmware-visible values as one `publish` per owner. The owners use + /// distinct queues and data masters. Every check precedes the first store, + /// so a refusal changes nothing; every table word is read before the first + /// store, so no uncached read waits for an earlier store to drain. + pub(crate) fn publish_owners( + &mut self, + view: &View<'_>, + owners: [Owner; N], + scheduler: u64, + policy: Policy, + clock: impl FnOnce() -> (u64, u64), + ) -> Result<[Publication; N]> { if scheduler == 0 { return Err(EINVAL); } - let next_qid = self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; - let next_dm = self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; - let starts_busy = self.dm_in_flight[dm] == 0; - let time = if starts_busy { + let mut indices = [(0, 0, 0); N]; + for (index, owner) in owners.iter().enumerate() { + indices[index] = owner.indices()?; + let (qid, _, dm) = indices[index]; + if indices[..index] + .iter() + .any(|&(other_qid, _, other_dm)| other_qid == qid || other_dm == dm) + { + return Err(EINVAL); + } + self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; + self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; + } + let rows = indices.map(|(qid, _, _)| view.row_word(qid)); + let queue_submitted = indices.map(|(qid, _, _)| view.queue_submitted[qid].load(Ordering::Relaxed)); + let owner_submitted = indices.map(|(_, qos, _)| view.owner_submitted[qos].load(Ordering::Relaxed)); + let dm_submitted = indices.map(|(_, _, dm)| view.times_and_counts[3 + dm].load(Ordering::Relaxed)); + let starts_busy = indices.map(|(_, _, dm)| self.dm_in_flight[dm] == 0); + let busy = starts_busy.iter().any(|&starts| starts); + let intervals = if busy { view.intervals.load(Ordering::Relaxed) } else { 0 }; + for (index, owner) in owners.iter().enumerate() { + let (qid, _, _) = indices[index]; + if self.in_flight[qid] != 0 + && (rows[index][0] != owner.qos || self.dm_by_qid[qid] != owner.data_master) + { + return Err(EBUSY); + } + } + let time = if busy { let (counter, frequency) = clock(); time_from_host(counter, frequency)? } else { 0 }; - if self.in_flight[qid] != 0 - && (view.rows[qid][0].load(Ordering::Relaxed) != owner.qos - || self.dm_by_qid[qid] != owner.data_master) - { - return Err(EBUSY); + let class = policy.qos_class(); + let mut first = false; + for (index, owner) in owners.iter().enumerate() { + let (qid, qos, _) = indices[index]; + view.rows[qid][0].store(owner.qos, Ordering::Relaxed); + view.rows[qid][1].store(class, Ordering::Relaxed); + view.scheduler[qos].store(scheduler, Ordering::Relaxed); + view.share[qos].store(policy.qos_share(), Ordering::Relaxed); + first |= self.in_flight[qid] == 0; } - view.rows[qid][0].store(owner.qos, Ordering::Relaxed); - view.rows[qid][1].store(policy.qos_class(), Ordering::Relaxed); - let record = view.row(qid); - view.scheduler[qos].store(scheduler, Ordering::Relaxed); - view.share[qos].store(policy.qos_share(), Ordering::Relaxed); - if self.in_flight[qid] == 0 { + if first { fence(Ordering::SeqCst); } - view.queue_submitted[qid].store( - view.queue_submitted[qid] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - view.owner_submitted[qos].store( - view.owner_submitted[qos] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - if starts_busy { - view.times_and_counts[dm].store(time, Ordering::Relaxed); - } - let submitted = &view.times_and_counts[3 + dm]; - let dm_submitted = submitted.load(Ordering::Relaxed).wrapping_add(1); - submitted.store(dm_submitted, Ordering::Relaxed); - if starts_busy { - view.intervals.store( - view.intervals.load(Ordering::Relaxed).wrapping_add(1), + let mut publications = [Publication { + record: [0; 8], + owner: owners[0], + dm_submitted: 0, + }; N]; + let mut intervals = intervals; + for (index, owner) in owners.iter().enumerate() { + let (qid, qos, dm) = indices[index]; + view.queue_submitted[qid].store(queue_submitted[index].wrapping_add(1), Ordering::Relaxed); + // Owners of one logical queue share their submission count. + let earlier = indices[..index].iter().filter(|&&(_, other, _)| other == qos).count(); + view.owner_submitted[qos].store( + owner_submitted[index].wrapping_add(earlier as u32 + 1), Ordering::Relaxed, ); + if starts_busy[index] { + view.times_and_counts[dm].store(time, Ordering::Relaxed); + } + let submitted = dm_submitted[index].wrapping_add(1); + view.times_and_counts[3 + dm].store(submitted, Ordering::Relaxed); + if starts_busy[index] { + intervals = intervals.wrapping_add(1); + view.intervals.store(intervals, Ordering::Relaxed); + } + let mut record = rows[index]; + record[0] = owner.qos; + record[1] = class; + self.in_flight[qid] += 1; + self.dm_in_flight[dm] += 1; + self.dm_by_qid[qid] = owner.data_master; + publications[index] = Publication { + record, + owner: *owner, + dm_submitted: submitted, + }; } - self.in_flight[qid] = next_qid; - self.dm_in_flight[dm] = next_dm; - self.dm_by_qid[qid] = owner.data_master; fence(Ordering::SeqCst); - Ok(Publication { - record, - owner, - dm_submitted, - }) + Ok(publications) } /// A possibly exposed producer must retain its reference until completion. diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 5c201ee6ec965..e8540faa620a1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -83,12 +83,13 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish( + /// Publishes `owners` in order with one pass over the QoS table. + fn qos_publish_pair( &mut self, - owner: qos::Owner, + owners: [qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result; + ) -> Result<[qos::Publication; 2]>; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; @@ -800,10 +801,12 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - qos_publications[1] = - Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); - qos_publications[0] = - Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); + let [fragment, tiling] = host.qos_publish_pair( + [qos_owners[1], qos_owners[0]], + context.scheduler_va(), + policy, + )?; + qos_publications = [Some(tiling), Some(fragment)]; // A class change happens only at a rebind, after the pair drained. if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 4aaad40eee765..b5649f059f5d2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -531,13 +531,13 @@ impl crate::g17::queue::render::Host for Host<'_> { core::sync::atomic::fence(Ordering::Acquire); Ok(stamps) } - fn qos_publish( + fn qos_publish_pair( &mut self, - owner: crate::g17::qos::Owner, + owners: [crate::g17::qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result { - self.firmware.port.qos_publish(owner, scheduler, policy) + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.firmware.port.qos_publish_pair(owners, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { self.firmware.port.qos_cancel(publication) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index cd7bad6ecce5d..dd3a9c881a142 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -53,13 +53,13 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { Ok([self.port.stamp(qids[0])?, self.port.stamp(qids[1])?]) } - fn qos_publish( + fn qos_publish_pair( &mut self, - owner: crate::g17::qos::Owner, + owners: [crate::g17::qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result { - self.port.qos_publish(owner, scheduler, policy) + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.port.qos_publish_pair(owners, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { self.port.qos_cancel(publication) From b86c445ba2829f644075ca663217abe65844e8c0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:59 -0400 Subject: [PATCH 45/64] drm/asahi: stop copying G17 fragment registers to the unused client page Every render preparation zeroed the pair's client register page (one uncached 16 KiB page) and copied the fragment descriptor's register arrays into it. Nothing reads that page: kick entries bind the register arrays embedded in the descriptor (register_bindings), and the page is kept only for its allocation position. Keep the array bounds check and drop the copy. Same-boot phase timers, render.light d1 pinned to cpu5: 426 -> 18 ns per submit. --- .../gpu/drm/asahi/g17/queue/render/memory.rs | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 80ee0b871cc82..b14ccbf879b5f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -571,21 +571,10 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - self.registers.bytes_mut().fill(0); - for (index, array) in fragment.descriptor.arrays.iter().enumerate() { - for (entry, value) in array - .entries - .get(..(array.size & 0xffff) as usize) - .ok_or(EINVAL)? - .iter() - .enumerate() - { - self.registers.write( - index * size_of::() - + entry * size_of::(), - *value, - )?; - } + // Kick entries bind the arrays embedded in this descriptor + // (register_bindings); the client register page is never read. + for array in &fragment.descriptor.arrays { + array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; } fragment.set_mcache( args.fragment.descriptor_va, From 5a55068193e213d7115a63f2222ffc304a4ced98 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:59 -0400 Subject: [PATCH 46/64] drm/asahi: read the G17 render work rings once per direct publication The direct render path read both render work rings' cursors (uncached) for the room check, again to write the pair's records and again to expose them, all under the render publication lock. No other host producer moves while that lock is held and the firmware consumers only advance, so the first reading stays valid. Return it from the room check and use its producers for the write and the exposure. Same-boot phase timers, render.light d1 pinned to cpu5: record write and exposure 435 + 161 -> 410 + 113 ns per submit. --- drivers/gpu/drm/asahi/g17/port.rs | 75 +++++++++++++------ .../drm/asahi/g17/runtime/render/direct.rs | 17 ++++- 2 files changed, 64 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 45ddae9719363..8d129c49c640c 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -103,6 +103,22 @@ pub(super) struct Port { submissions: AtomicU64, } +/// Room in both render work rings of one priority, observed under the render publication +/// lock. While that lock is held no other host producer moves, and the firmware consumers only +/// advance, so the observed producers stay current and the room stays available. +#[derive(Clone, Copy)] +pub(super) struct RenderRoom { + priority: u8, + /// Fragment, then tiling producer. + producers: [u32; 2], +} + +impl RenderRoom { + pub(super) fn priority(&self) -> u8 { + self.priority + } +} + /// A compute work-ring slot reserved before any command becomes visible. Dropping an unused /// claim returns the slot. pub(super) struct OuterClaim<'a> { @@ -225,20 +241,33 @@ impl Port { )) } - /// Both render work rings of `priority` have a free slot. Only firmware consumption, - /// never a publisher without the render lock, changes this. - pub(super) fn render_room(&self, _guard: &MutexGuard<'_, ()>, priority: u8) -> Result { + /// Reads both render work rings of `priority` once. Returns their producers if both have a + /// free slot; the room holds for as long as `guard`. Only firmware consumption, never a + /// publisher without the render lock, changes it. + pub(super) fn observe_render_room( + &self, + _guard: &MutexGuard<'_, ()>, + priority: u8, + ) -> Result> { let (fragment, tiling) = self.render_rings(priority)?; - Ok(fragment.free()? > 0 && tiling.free()? > 0) + let cursors = [fragment.cursors()?, tiling.cursors()?]; + let room = |[first, second, producer]: [u32; 3]| { + let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + room(first).min(room(second)) + }; + Ok((room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { + priority, + producers: [cursors[0][2], cursors[1][2]], + })) } - /// Copies a render pair's fragment, then tiling record into the slots after each - /// producer without exposing them, as `InitData::publish_work_pair` does for a deferred - /// pair. Returns the producers that expose them. + /// Copies a render pair's fragment, then tiling record into the slots after the producers + /// that `room` observed, without exposing them, as `InitData::publish_work_pair` does for a + /// deferred pair. Returns the producers that expose them. pub(super) fn write_render_pair( &self, _guard: &MutexGuard<'_, ()>, - priority: u8, + room: &RenderRoom, fragment: &WorkSlot, tiling: &WorkSlot, commit_accounting: impl FnOnce() -> Result, @@ -249,21 +278,20 @@ impl Port { { return Err(EINVAL); } - let (fragment_ring, tiling_ring) = self.render_rings(priority)?; - let slot = |ring: &RingView| -> Result<(*mut u8, u32)> { - let [first, second, producer] = ring.cursors()?; - let next = (producer + 1) % WORK_RING_SLOTS; - if next == first || next == second { + let (fragment_ring, tiling_ring) = self.render_rings(room.priority)?; + let slot = |ring: &RingView, producer: u32| -> Result<(*mut u8, u32)> { + if producer >= WORK_RING_SLOTS { return Err(EINVAL); } // SAFETY: The ring view spans WORK_RING_SLOTS records; `producer` is in range. - Ok((unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }, next)) + let slot = unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }; + Ok((slot, (producer + 1) % WORK_RING_SLOTS)) }; - let (fragment_slot, fragment_next) = slot(&fragment_ring)?; - let (tiling_slot, tiling_next) = slot(&tiling_ring)?; + let (fragment_slot, fragment_next) = slot(&fragment_ring, room.producers[0])?; + let (tiling_slot, tiling_next) = slot(&tiling_ring, room.producers[1])?; let copy = |record: &WorkSlot, destination: *mut u8| { - // SAFETY: The slot after the producer is host-owned until the producer moves past - // it, and the render lock excludes every other host writer of this ring. + // SAFETY: The room proves that the slot after the producer is host-owned until the + // producer moves past it, and the render lock excludes every other host writer. unsafe { core::ptr::copy_nonoverlapping( core::ptr::from_ref(record).cast::(), @@ -281,17 +309,16 @@ impl Port { Ok([fragment_next, tiling_next]) } - /// Exposes a pair written by `write_render_pair`, fragment first, as - /// `InitData::publish_retained_work_pair` does: the producers must still precede `next`. + /// Exposes a pair written by `write_render_pair` with `room`, fragment first, as + /// `InitData::publish_retained_work_pair` does: `next` must follow the observed producers. pub(super) fn expose_render_pair( &self, _guard: &MutexGuard<'_, ()>, - priority: u8, + room: &RenderRoom, next: [u32; 2], ) -> Result { - let (fragment_ring, tiling_ring) = self.render_rings(priority)?; - for (ring, next) in [(&fragment_ring, next[0]), (&tiling_ring, next[1])] { - let [_, _, producer] = ring.cursors()?; + let (fragment_ring, tiling_ring) = self.render_rings(room.priority)?; + for (producer, next) in room.producers.into_iter().zip(next) { if next != (producer + 1) % WORK_RING_SLOTS { return Err(EBUSY); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs index dd3a9c881a142..1f42d1e6f404f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -14,7 +14,7 @@ use super::super::{Backend, DeferredBatch}; use crate::g17::{ job::Packet, object::Allocator, - port::Port, + port::{Port, RenderRoom}, queue::render::{Pair, PublicationLease, RenderAddresses}, }; use core::sync::atomic::Ordering; @@ -39,6 +39,7 @@ pub(in crate::g17::runtime) struct Loan { struct DirectHost<'a> { port: &'a Port, render: &'a MutexGuard<'a, ()>, + room: RenderRoom, generation: u64, activation: Option<(u16, u64)>, } @@ -79,8 +80,11 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { commit: impl FnOnce() -> Result, late_tiling: impl FnOnce(), ) -> Result<[u32; 2]> { + if priority != self.room.priority() { + return Err(EINVAL); + } self.port - .write_render_pair(self.render, priority, fragment, tiling, commit, late_tiling) + .write_render_pair(self.render, &self.room, fragment, tiling, commit, late_tiling) } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { // The lease leaves preparation when the pair is returned; nothing can @@ -95,7 +99,10 @@ impl crate::g17::queue::render::Host for DirectHost<'_> { _tiling: &crate::g17::fw::channels::WorkSlot, next: [u32; 2], ) -> Result { - self.port.expose_render_pair(self.render, priority, next) + if priority != self.room.priority() { + return Err(EINVAL); + } + self.port.expose_render_pair(self.render, &self.room, next) } /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { @@ -302,9 +309,10 @@ impl Backend { }; pair.prepare(&alloc, packet, loan.addresses)?; let render = port.render_lock(); - if !lease.is_current() || !port.render_room(&render, priority)? { + if !lease.is_current() { return Err(EBUSY); } + let room = port.observe_render_room(&render, priority)?.ok_or(EBUSY)?; let mut dependencies = pair.own_dependencies(self.owner, packet)?; if let Some(footprint) = packet.footprint.as_ref() { dependencies.hazard_count = self.admission.publication_barriers( @@ -317,6 +325,7 @@ impl Backend { let mut host = DirectHost { port, render: &render, + room, generation, activation: None, }; From b67018ed5974069c13c13249834dbc3b84e806ed Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 05:46:59 -0400 Subject: [PATCH 47/64] drm/asahi: request the G17 tiling descriptor slot before preparing a render Clearing a render's tiling descriptor slot took about 480 ns for 2.5 KiB while clearing the 8.7 KiB fragment slot took 80 ns: the slot was last used by the firmware for an earlier command, and every line had to be fetched for writing one at a time. Issue store prefetches for the slot at the start of preparation, so the transfers overlap the target, scene and generation preparation that precedes the descriptor write. Same-boot phase timers, render.light d1 pinned to cpu5: tiling descriptor clear 483 -> 52 ns, ioctl entry to doorbell 12.04 -> 11.39 us. Same-boot A/B of this series (exp_mask diagnostic restoring each old path, 3 interleaved rounds, gpubench --runs 5 --run-ms 250, us/submit): render.light d1/d2/d32 cpu5 126.5/48.5/31.7 -> 106.1/46.7/30.8, cpu0 104.7/46.6/44.7 -> 103.6/46.0/37.5 (d32 cpu0 noisy); render ioctl entry to doorbell 14.4 -> 12.1 us (phase timers). compute.light, render queues d2/4/8, render procs d2/16, copy.bw buf2img d1/4 and sync.signal hop unchanged within noise. --- drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + .../gpu/drm/asahi/g17/queue/render/memory.rs | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index e8540faa620a1..b9f179495ab99 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -374,6 +374,7 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; + self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index b14ccbf879b5f..e7d835e15eb47 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -537,6 +537,26 @@ const _: () = { }; impl Memory { + /// Requests the tiling descriptor slot of `ordinal` for writing. The firmware accessed + /// that slot for the command that used it last, so the host no longer holds its lines; + /// requesting them before the rest of the preparation overlaps those transfers with it + /// instead of stalling the descriptor write one line at a time. + pub(super) fn prefetch_descriptors(&self, ordinal: u64) { + const LINE: usize = 64; + let size = size_of::(); + let Ok(base) = self.graph.descriptors.pointer(Graph::offset(0, ordinal), size) else { + return; + }; + for offset in (0..size).step_by(LINE) { + // SAFETY: A prefetch hint inside the retained mapping; it has no + // architectural effect on memory. + unsafe { + core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(offset), + options(nostack, preserves_flags, readonly)); + } + } + } + pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, From 4d0362d0c18d288d3bd520fdfb69c2e7624cf6b0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 06:04:01 -0400 Subject: [PATCH 48/64] drm/asahi: store G17 register arrays without an intermediate copy Without scratch storage a render's first tiling and fragment register arrays drop their SCS-enable entry. Each was copied into a trimmed array on the stack, initialised entry by entry, and then copied again into the descriptor. Store the remaining entries straight into the descriptor's array instead; the stored entries, size and address are unchanged. Same-boot phase timers, render.light d1 pinned to cpu5: tiling register store 310 -> 45 ns, fragment array 0 store 245 -> 69 ns, ioctl entry to doorbell 11.15 -> 10.77 us. --- drivers/gpu/drm/asahi/g17/fw/kick.rs | 14 ++++++++++++++ drivers/gpu/drm/asahi/g17/fw/render.rs | 9 ++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 4f3a70671eb87..ca2b6604a302b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -278,6 +278,20 @@ impl RegisterArray { self.size = (size_of_val(writes) as u32) << 16 | count as u32; Ok(()) } + + /// Stores `writes` without the entry at `skip`, exactly as [`Self::set`] stores the + /// remaining writes, without first copying them elsewhere. + pub(crate) fn set_without(&mut self, self_va: u64, writes: &[RegisterWrite], skip: usize) -> Result { + let count = writes.len().checked_sub(1).ok_or(EINVAL)?; + if skip > count || count > REGISTER_ARRAY_ENTRIES { + return Err(EINVAL); + } + self.entries[..skip].copy_from_slice(&writes[..skip]); + self.entries[skip..count].copy_from_slice(&writes[skip + 1..]); + self.self_va = self_va; + self.size = ((count * size_of::()) as u32) << 16 | count as u32; + Ok(()) + } } /// Binding of one [`RegisterArray`] to a kick entry. diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 0e270d40392a8..aa8693bf7140b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -1110,9 +1110,7 @@ impl TaDescriptor { } else if scs_register { self.registers.set(array_va, ®isters)?; } else { - let mut trimmed = [registers[0]; TA_REGISTERS]; - trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); - self.registers.set(array_va, &trimmed)?; + self.registers.set_without(array_va, ®isters, TA_SCS_ENABLE_INDEX)?; } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; @@ -1456,10 +1454,7 @@ impl FragmentSlot { } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { - let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; - trimmed[FRAGMENT_SCS_ENABLE_INDEX..] - .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); - desc.arrays[0].set(arrays[0], &trimmed)?; + desc.arrays[0].set_without(arrays[0], ®isters, FRAGMENT_SCS_ENABLE_INDEX)?; } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; From 861b39d948f6c0588b3646e1246e660b52230e5b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 06:04:01 -0400 Subject: [PATCH 49/64] drm/asahi: also request the G17 queue configuration records early A render's two queue configuration records are rewritten for every command that carries one, in records the firmware accessed for an earlier command, the same pattern as the tiling descriptor slot. Request them for writing together with that slot at the start of preparation. Same-boot phase timers, render.light d1 pinned to cpu5: fragment prefix and kick 219 -> 176 ns, record write 328 -> 271 ns, ioctl entry to doorbell 10.77 -> 10.55 us. --- .../gpu/drm/asahi/g17/queue/render/memory.rs | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index e7d835e15eb47..756f96e3fd048 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -537,22 +537,30 @@ const _: () = { }; impl Memory { - /// Requests the tiling descriptor slot of `ordinal` for writing. The firmware accessed - /// that slot for the command that used it last, so the host no longer holds its lines; - /// requesting them before the rest of the preparation overlaps those transfers with it - /// instead of stalling the descriptor write one line at a time. + /// Requests the tiling descriptor slot and both queue configuration records of + /// `ordinal` for writing. The firmware accessed them for the command that used them + /// last, so the host no longer holds their lines; requesting them before the rest of + /// the preparation overlaps those transfers with it instead of stalling the writes one + /// line at a time. pub(super) fn prefetch_descriptors(&self, ordinal: u64) { - const LINE: usize = 64; - let size = size_of::(); - let Ok(base) = self.graph.descriptors.pointer(Graph::offset(0, ordinal), size) else { - return; - }; - for offset in (0..size).step_by(LINE) { - // SAFETY: A prefetch hint inside the retained mapping; it has no - // architectural effect on memory. - unsafe { - core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(offset), - options(nostack, preserves_flags, readonly)); + let record = ordinal as usize % RECORDS; + let config = size_of::(); + let ranges = [ + (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), + (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), + (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), + ]; + for (object, offset, size) in ranges { + let Ok(base) = object.pointer(offset, size) else { + continue; + }; + for line in (0..size).step_by(64) { + // SAFETY: A prefetch hint inside the retained mapping; it has no + // architectural effect on memory. + unsafe { + core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(line), + options(nostack, preserves_flags, readonly)); + } } } } From 66ef051d8bd4e66bdc145eb7fb7a920235a8e668 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 06:04:01 -0400 Subject: [PATCH 50/64] drm/asahi: scan only render pool slots for G17 free-list leases Render free-list leases live only in slots from FIRST_RENDER_POOL up; Pools::slot() refuses every lower index. The release observation each render preparation performs, the release publication and the linger expiry still walked all 256 entries. Walk the render range only. Phase timers, render.light d1 pinned to cpu5 (two boots): lease observation 346 -> 213 ns per submit. --- drivers/gpu/drm/asahi/g17/runtime/render/pools.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 78e5e3a219358..1e3354d90dbfb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -179,7 +179,7 @@ impl Pools { ) -> (bool, Option) { let mut ended = false; let mut next: Option = None; - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { let Some(since) = lease.lingering() else { continue; }; @@ -212,7 +212,7 @@ impl Pools { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { lease.observe_releases(consumer, producer)?; } Ok(()) @@ -225,7 +225,7 @@ impl Pools { init: &InitData, mut announce: impl FnMut() -> Result, ) -> Result { - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { let Some(release) = lease.pending_release() else { continue; }; From 1aec5be232da7bacbbe38b7f9b22c27906eac8c6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 07:49:37 -0400 Subject: [PATCH 51/64] drm/asahi: omit G17 render queue configurations that only reload the held completion A render published onto a pair whose queues have drained seeds each queue's completion value, and the seed forces a queue configuration record onto both item rings. When the previous publication of the pair in the same recovery epoch was the last kick of each queue, the queue already holds exactly the value the seed would load, so the record carries nothing new. Keep the completion values of each successful publication with the configuration cache and publish the record only when its identity changed or the seed differs from the held value. Recovery, replay and graph rebuilds drop the kept values, so those publications still seed. The firmware processes each record before starting the kick. On Neo it shortened the kick of a render on a drained pair by about 1.5 us per record (start-to-start 13.5 -> 10.7 us in render.light queues d4). Same-boot A/B, alternated per row, 3 rounds, medians in us per submit: render.light queues d2 54.70 -> 51.88 (-5.2%) render.light queues d4 45.25 -> 42.15 (-6.8%) render.light queues d8 41.65 -> 38.79 (-6.9%) render.light procs d2 54.68 -> 51.84 (-5.2%) render.light procs d16 40.84 -> 37.63 (-7.9%) Deep single-queue rows publish onto busy pairs, which never seed, and are unchanged. --- drivers/gpu/drm/asahi/g17/queue/render.rs | 31 ++++++++++++++++--- .../gpu/drm/asahi/g17/queue/render/parked.rs | 1 + 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index b9f179495ab99..9469d526f49c8 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -192,6 +192,9 @@ pub(crate) struct Pair { /// An owner change in flight: the previous scheduler page and the ordinal of /// the new owner's first command, until that command leaves the pair. switch: Option<(u64, u64)>, + /// Completion values of the two kicks of the last publication, kept with the configuration + /// cache: what each queue holds once that publication completed. + published_completions: Option<[u64; 2]>, } impl Pair { #[inline(never)] @@ -290,6 +293,7 @@ impl Pair { context_update: true, executed: 0, switch: None, + published_completions: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -649,10 +653,26 @@ impl Pair { } else { [None; 2] }; - let present = [ - publication::required(cache[0], configs[0]), - publication::required(cache[1], configs[1]), - ]; + // A covered queue holds the completion value of its newest kick. When that kick is the + // previous publication of this pair in this recovery epoch, the seed would reload the + // value already held, so the configuration record is needed only for its identity. + // Each record costs the firmware time before it starts the kick. + let held = self + .published_completions + .take() + .filter(|_| self.recovery_generation == epoch); + let present = [0, 1].map(|stage| { + let mut config = configs[stage]; + let reload = kicks[stage].prev().completion_value(payload.wrapping_sub(1)); + if config.completion_seed_valid != 0 + && self.installed[stage] + && held.is_some_and(|held| held[stage] == reload) + { + config.completion_seed_valid = 0; + config.completion_seed = 0; + } + publication::required(cache[stage], config) + }); let cursors = self.memory.graph.cursors()?; if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); @@ -865,6 +885,7 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; + self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); @@ -1142,6 +1163,7 @@ impl Pair { pub(crate) fn resync(&mut self) { self.configs = [None; 2]; + self.published_completions = None; for kick in &mut self.kicks { kick.clear_parent_after_recovery(); } @@ -1417,6 +1439,7 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; packet.reset_publication_for_replay(); + self.published_completions = None; self.replay.push(packet).map_err(|_| EIO)?; } Ok(true) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 209ce0b8176ec..81d4306eda28c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -252,6 +252,7 @@ impl ParkedPair { context_update: true, executed: self.executed, switch: None, + published_completions: None, })) } } From d0df244b8f023037bb0f9cc9793f1d0f285cfc87 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 08:31:47 -0400 Subject: [PATCH 52/64] drm/asahi: keep one CPU mapping per G17 timestamp buffer Every timestamped command mapped the destination buffer into the kernel for each timestamp it carried: owned_vmap() takes the object's reservation lock to count a mapping user. Because dropping that mapping takes the lock too, which fence producers must not wait for, each command also allocated a cleanup work item and queued it when its completion was released, waking a worker per command. Map a timestamp buffer once, when it is bound, and keep that mapping for the buffer's lifetime. A command's destination keeps a reference to the buffer and an offset, so releasing it takes no lock. The buffer's own last reference can still be a command's, released where fence producers run, so the buffer defers the release of its mappings to the cleanup queue; that happens once per buffer instead of once per command. The mapping is made outside the device mutex. Destination validation and the write itself are unchanged; G16's CPU-retired buffers keep their per-command mapping. Same-boot phase timers (exp_mask diagnostic restoring the old path), render.light d1 pinned to cpu5: timestamp destination resolve 410 -> 132 ns, completion teardown at the end of the ioctl 578 -> 153 ns, ioctl 16.9 -> 16.2 us with the timers; compute.light d1 resolve 367 -> 152 ns; query kind=occlusion d1 resolve 337 -> 141 ns, teardown 636 -> 162 ns. Same-boot ABBA, 4 rounds (query occlusion d1/d16, query timestamp d1/d16, render.light d1/d2/d32, compute.light d1/d2/d64, buf2img d1, sync.signal hop, idle.wake), pinned to cpu5: unchanged within noise. Regression gate passes; deqp-vk query_pool.*, synchronization2 basic and timeline_semaphore, api.command_buffers and renderpass subsets: 22847 pass, 791 not supported, 0 fail. --- drivers/gpu/drm/asahi/g17/completion.rs | 36 ++++------ drivers/gpu/drm/asahi/g17/mod.rs | 28 ++++---- drivers/gpu/drm/asahi/mmu.rs | 93 ++++++++++++++++++------- drivers/gpu/drm/asahi/timestamp.rs | 44 ++++++++++-- 4 files changed, 135 insertions(+), 66 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 11b4c323d4668..02e7bbc5c33dc 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -27,9 +27,9 @@ mod publication; pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { - // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, - cpu: mmu::TimestampMapping, + // The buffer keeps its GPU alias and its CPU mapping alive. + buffer: Arc, + offset: usize, } impl Destination { @@ -45,13 +45,13 @@ impl Destination { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { + if offset.checked_add(size_of::()).ok_or(EINVAL)? > buffer.size() { return Err(ERANGE); } - let cpu = mapping.timestamp_mapping(offset)?; - Ok(Some(Self { _mapping: mapping, cpu })) + let offset = buffer.timestamp_offset(offset)?; + Ok(Some(Self { buffer, offset })) } fn write(&self, ticks: u64) { @@ -61,13 +61,14 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.cpu.write(ns); + self.buffer.write_timestamp(self.offset, ns); barrier(Ordering::Release); } } -/// Timestamp mappings resolved and retained before any command is enqueued. -pub(crate) struct Destinations(Option; 4]>>); +/// Timestamp destinations resolved and retained before any command is enqueued. Dropping +/// one releases only a buffer reference; the buffer defers its own mapping release. +pub(crate) struct Destinations([Option; 4]); impl Destinations { pub(crate) fn resolve( @@ -82,22 +83,13 @@ impl Destinations { destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; } - // Empty submissions need no cleanup allocation. Timestamped commands - // prepare the unique work item before any output fence is published. - let retained = if destinations.iter().any(Option::is_some) { - Some(crate::cleanup::Deferred::new(destinations)?) - } else { - None - }; - Ok(Self(retained)) + Ok(Self(destinations)) } - pub(crate) fn has_outputs(&self) -> bool { self.0.is_some() } + pub(crate) fn has_outputs(&self) -> bool { self.0.iter().any(Option::is_some) } fn iter(&self) -> core::slice::Iter<'_, Option> { - let destinations: &[Option] = - self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); - destinations.iter() + self.0.iter() } } diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 8c6891785f1c2..c81760ad4646b 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1639,18 +1639,22 @@ impl gpu::Gpu for Gpu { mut bo: gem::ObjectRef, range: Range, ) -> Result { - let state = self.shared.state.lock(); - let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - crate::timestamp::Buffer::firmware(bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - addresses, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - )?) + let mapping = { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + )? + }; + // The buffer's CPU mapping takes the reservation lock: not under the device mutex. + crate::timestamp::Buffer::firmware(mapping) } } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 6a80d1b484b34..f5b079e9b5382 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1201,56 +1201,83 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// Prepared CPU timestamp access; destruction happens after completion signalling. -pub(crate) struct TimestampMapping { - mapping: kernel::drm::gem::shmem::VMapOwned, - offset: usize, -} +/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's +/// reservation lock, so it must not be dropped where fence producers run. +pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); -impl TimestampMapping { - /// Prepare one bounded CPU timestamp destination on retained GEM backing. - pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { +impl TimestampVmap { + pub(crate) fn new(gem: &gem::Object) -> Result { + Ok(Self(gem.owned_vmap::<0>()?)) + } + + fn size(&self) -> usize { use kernel::drm::gem::BaseObject; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - Ok(Self { - mapping: gem.owned_vmap::<0>()?, - offset, - }) + self.0.owner().size() } - pub(crate) fn write(&self, value: u64) { + /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete + /// atomic store; byte stores support unaligned destinations without creating mutable + /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused + /// when the destination is prepared, and ignored here. + pub(crate) fn write(&self, offset: usize, value: u64) { + if offset.checked_add(size_of::()).is_none_or(|end| end > self.size()) { + return; + } // Completion runs in task context. Bindings of the same BO share this lock, // so aligned u64 and overlapping unaligned byte stores never race in Rust. - let _guard = self.mapping.owner().timestamp_lock.lock(); - // SAFETY: timestamp_mapping checked the complete extent inside this pinned vmap. - let base = self.mapping.addr.cast::(); - let pointer = unsafe { base.add(self.offset) }; + let _guard = self.0.owner().timestamp_lock.lock(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. + let base = self.0.addr.cast::(); + let pointer = unsafe { base.add(offset) }; if pointer as usize % core::mem::align_of::() == 0 { - // SAFETY: timestamp_mapping checked all eight bytes; the vmap pins the backing. + // SAFETY: All eight bytes were checked above; the vmap pins the backing. // A naturally aligned query result becomes available in one complete store. let destination = unsafe { &*pointer.cast::() }; destination.store(value.to_le(), Ordering::Relaxed); } else { for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { &*base.add(self.offset + index).cast::() }; + // SAFETY: The complete eight-byte extent was checked above. The owned + // vmap pins its backing; byte atomics permit unaligned and overlapping + // timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(offset + index).cast::() }; destination.store(byte, Ordering::Relaxed); } } // A CPU-retired query can be consumed by a later GPU command. WRITEBACK // objects use a cacheable vmap, while GPU mappings need not snoop that cache. // Publish the bytes before signalling the completion fence or queue frontier. - if self.mapping.owner().is_writeback() { + if self.0.owner().is_writeback() { crate::pgtable::clean_poc(pointer, size_of::()); } crate::pgtable::poc_barrier(); } } +/// Prepared CPU timestamp access with its own vmap; destruction happens after completion +/// signalling. +pub(crate) struct TimestampMapping { + mapping: TimestampVmap, + offset: usize, +} + +impl TimestampMapping { + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + Ok(Self { + mapping: TimestampVmap::new(gem)?, + offset, + }) + } + + pub(crate) fn write(&self, value: u64) { + self.mapping.write(self.offset, value); + } +} + /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. pub(crate) struct KernelMapping(ManuallyDrop>); @@ -1286,6 +1313,13 @@ impl KernelMapping { /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + TimestampMapping::new(gem, self.timestamp_offset(offset)?) + } + + /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete + /// extent lies inside both the mapping and the object. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1296,7 +1330,12 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - TimestampMapping::new(gem, start) + Ok(start) + } + + /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. + pub(crate) fn timestamp_vmap(&self) -> Result { + TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 976ec9f6bf48b..0d365a8d930bb 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -13,10 +13,20 @@ use kernel::{prelude::*, sync::Arc}; #[cfg(CONFIG_DRM_ASAHI_G16)] use kernel::{uapi, xarray}; +/// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the +/// buffer's lifetime so that commands need no mapping of their own. +pub(crate) struct FirmwareBuffer { + mapping: Arc, + cpu: mmu::TimestampVmap, +} + /// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. /// CPU-retired timestamps retain their backing without consuming global firmware address space. pub(crate) enum Buffer { - Firmware(Arc), + /// The last reference may be a command's, dropped where fence producers run, while + /// releasing the CPU mapping takes the reservation lock: the deferral runs the release + /// on the cleanup queue. + Firmware(crate::cleanup::Deferred), #[cfg(CONFIG_DRM_ASAHI_G16)] Cpu { // The GEM pin keeps the object's held SG table alive through CPU publication. @@ -27,7 +37,11 @@ pub(crate) enum Buffer { impl Buffer { pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { - Ok(Self::Firmware(Arc::new(mapping, GFP_KERNEL)?)) + let cpu = mapping.timestamp_vmap()?; + Ok(Self::Firmware(crate::cleanup::Deferred::new(FirmwareBuffer { + mapping: Arc::new(mapping, GFP_KERNEL)?, + cpu, + })?)) } /// Retain a CPU-written byte range without publishing any firmware address. @@ -46,7 +60,7 @@ impl Buffer { pub(crate) fn size(&self) -> usize { match self { - Self::Firmware(mapping) => mapping.size(), + Self::Firmware(buffer) => buffer.mapping.size(), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { range, .. } => range.end - range.start, } @@ -54,7 +68,7 @@ impl Buffer { pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { match self { - Self::Firmware(mapping) => mapping.timestamp_mapping(offset), + Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { gem, range, .. } => { if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -65,10 +79,30 @@ impl Buffer { } } + /// The object offset of the timestamp at `offset` of a firmware buffer, for + /// [`Self::write_timestamp`]. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { + match self { + Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } + + /// Writes a timestamp at an object offset returned by [`Self::timestamp_offset`], + /// through the buffer's own CPU mapping. + pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { + match self { + Self::Firmware(buffer) => buffer.cpu.write(offset, value), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => {} + } + } + /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. pub(crate) fn firmware_mapping(&self) -> Result> { match self { - Self::Firmware(mapping) => Ok(mapping.clone()), + Self::Firmware(buffer) => Ok(buffer.mapping.clone()), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { .. } => Err(EINVAL), } From 3ff8c49c5f12b908a72e6c076ff0d7336ffddcb8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 12:55:25 -0400 Subject: [PATCH 53/64] drm/asahi: withhold G17 compute doorbells behind two unretired commands Each compute publication rang its pipe doorbell. The firmware handles a doorbell by reading the whole work ring of the priority class, and a doorbell that arrives while it finishes a command delays that command's completion: writing its end timestamp and stamp, and releasing the commands that wait for it. In the cross-queue hop the stamp followed the GPU end by 11.5 us without a doorbell in that window and by about 24 us with one. Three quarters of the completions had one, because each hop publishes two commands. A command with two or more unretired commands of its queue ahead cannot be the next one to start. Withhold its doorbell, and ring the withheld doorbell from the compute scan once at most one unretired command precedes the first publication it covers. A doorbell rung meanwhile on the same ring announces the work slot earlier. Shallow queues ring as before. Same-boot A/B on Neo, alternated per row, 3 rounds, unpinned, medians: sync.signal queue timeline 30.13 -> 23.60 us per hop (-21.7%) sync.signal queue binary 29.14 -> 23.31 us per hop (-20.0%) compute.light d64 14.55 -> 13.91 us (-4.4%) compute.light d1/d2 unchanged (103.0 / 34.0 us) render.light d1/d32 unchanged Firmware side, from the completion words: GPU end to stamp went from a median of 24.3 us to 12.0 us, and doorbells per command from 1.0 to 0.2. --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 41 ++++++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 10 ++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index c8e00e2b563c2..5359a732562c4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -369,6 +369,10 @@ use crate::g17::{ }; use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; +/// Position in its queue's active ring from which a publication withholds its doorbell: +/// with two unretired commands ahead, it cannot be the next to start. +const WITHHOLD_DEPTH: usize = 3; + struct Fifo { entries: [Option; IN_FLIGHT], head: usize, @@ -589,9 +593,22 @@ pub(crate) struct Queue { retirement_proved: bool, source_readers: Arc, source_hint: crate::g17::sync_input::ReaderHint, + /// The doorbell withheld from publications that had [`WITHHOLD_DEPTH`] - 1 or more + /// unretired commands of this queue ahead of them, and the publication of the first. + withheld_doorbell: Option<(u64, u64)>, } impl Queue { + /// Returns the withheld doorbell once at most one unretired command of this queue + /// precedes the first publication it covers. + pub(crate) fn due_doorbell(&mut self) -> Option { + let (doorbell, first) = self.withheld_doorbell?; + if self.active.iter().filter(|active| active.published < first).count() > 1 { + return None; + } + self.withheld_doorbell = None; + Some(doorbell) + } /// The caller reserves a QID first, cancels it on construction failure, and retains every /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( @@ -656,6 +673,7 @@ impl Queue { retirement_proved: false, source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, source_hint: crate::g17::sync_input::ReaderHint::new(), + withheld_doorbell: None, }) } pub(crate) fn qid(&self) -> u8 { @@ -1599,6 +1617,7 @@ impl Queue { ring: DescriptorRing, ) -> Result<(u16, u64)> { if !self.can_rearm() { return Err(EAGAIN); } + self.withheld_doorbell = None; self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; @@ -2226,12 +2245,30 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - // Every publication rings its own pipe and priority class, as render + // A publication rings its own pipe and priority class, as render // publication does: pipe 2 (compute) in bits 1:0, priority class in // bits 3:2. The generic work doorbell left the firmware to find the // ring itself and delayed dispatch. + // + // A doorbell makes the firmware read the whole work ring of its class, + // and each one it handles while finishing a command delays that + // command's completion and the release of its waiters. A command with + // two or more unretired commands of its queue ahead cannot start soon: + // its doorbell is withheld until at most one precedes it (see + // `due_doorbell`), and any doorbell rung meanwhile on the same ring + // announces its work slot earlier. fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; + let doorbell = (0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2; + if self.active.len >= WITHHOLD_DEPTH + && self.withheld_doorbell.is_none_or(|(withheld, _)| withheld == doorbell) + { + self.withheld_doorbell.get_or_insert((doorbell, published)); + } else { + host.notify(doorbell)?; + if self.withheld_doorbell.is_some_and(|(withheld, _)| withheld == doorbell) { + self.withheld_doorbell = None; + } + } crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 7f89a25ae6d35..625339afafeaa 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1661,6 +1661,7 @@ impl super::Firmware { let port = &self.port; let retired = deferred.pending(); let mut freed = false; + let mut doorbells = [0u64; 4]; for mut queue in registry .compute .iter_mut() @@ -1677,6 +1678,15 @@ impl super::Firmware { &mut |event| deferred.push(event), )?; freed |= busy && queue.channel_free(); + // Withheld doorbells are rung once their command is next but one; one per + // priority class announces every queue's work slots on that ring. + if let Some(doorbell) = queue.due_doorbell() { + let class = ((doorbell >> 2) & 3) as usize; + doorbells[class] = doorbell; + } + } + for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { + self.primary.notify(doorbell)?; } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. From b1f5dc6df0ba5650c8424027a025db233c6544c4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 13:07:37 -0400 Subject: [PATCH 54/64] drm/asahi: drop G17 compute doorbells that another doorbell announced A withheld compute doorbell is rung once its command is next but one. When a doorbell of the same priority class was rung after the last withheld work slot was written, the firmware has already read that slot, so the withheld doorbell repeats an announcement it has handled. Count the compute work slots written per class, and note the count read before each compute doorbell is rung. A due withheld doorbell whose slots are covered by that count is dropped instead of rung. Same-boot A/B on Neo against the withholding alone, alternated per row, 3 rounds, unpinned, medians: sync.signal queue timeline 23.47 -> 22.70 us per hop (-3.3%) sync.signal queue binary 23.41 -> 23.02 us per hop (-1.7%) compute.light d1/d2/d64, render.light d1/d32: unchanged Doorbells per command in the hop went from 0.22 to 0.16. --- drivers/gpu/drm/asahi/g17/port.rs | 32 ++++++++++++++++++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 29 ++++++++++++++------ drivers/gpu/drm/asahi/g17/runtime.rs | 16 +++++++++-- 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs index 8d129c49c640c..db043ef9d6f91 100644 --- a/drivers/gpu/drm/asahi/g17/port.rs +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -98,6 +98,11 @@ pub(super) struct Port { generation: AtomicU64, /// Device publication order shared by compute and render commands. publications: AtomicU64, + /// Compute work slots written per priority class. + compute_written: [AtomicU64; PRIORITIES], + /// `compute_written` as read before the latest compute doorbell of each class: every slot + /// counted there was in its ring when the firmware handled that doorbell. + compute_announced: [AtomicU64; PRIORITIES], /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. effort_full: AtomicBool, submissions: AtomicU64, @@ -147,6 +152,8 @@ impl Port { render <- new_mutex!((), "g17::Port::render"), generation: AtomicU64::new(0), publications: AtomicU64::new(0), + compute_written: [const { AtomicU64::new(0) }; PRIORITIES], + compute_announced: [const { AtomicU64::new(0) }; PRIORITIES], effort_full: AtomicBool::new(false), submissions: AtomicU64::new(0), }), @@ -223,10 +230,35 @@ impl Port { unsafe { (*ring.producer).store(next, Ordering::Relaxed) }; fence(Ordering::SeqCst); reserved[claim.priority] = reserved[claim.priority].saturating_sub(1); + self.compute_written[claim.priority].fetch_add(1, Ordering::AcqRel); claim.armed = false; Ok(next) } + /// Compute work slots written at `priority` so far. + pub(super) fn compute_written(&self, priority: u8) -> u64 { + self.compute_written + .get(usize::from(priority)) + .map_or(u64::MAX, |written| written.load(Ordering::Acquire)) + } + + /// Compute work slots at `priority` that a doorbell has announced. + pub(super) fn compute_announced(&self, priority: u8) -> u64 { + self.compute_announced + .get(usize::from(priority)) + .map_or(0, |announced| announced.load(Ordering::Acquire)) + } + + /// Rings the compute doorbell `message` of `priority`, after noting which slots it + /// announces. + pub(super) fn notify_compute(&self, priority: u8, message: u64) -> Result { + let index = usize::from(priority); + let written = self.compute_written.get(index).ok_or(EINVAL)?.load(Ordering::Acquire); + self.notify(message)?; + self.compute_announced[index].fetch_max(written, Ordering::AcqRel); + Ok(()) + } + /// Taken around every render publication; see `render`. pub(super) fn render_lock(&self) -> MutexGuard<'_, ()> { self.render.lock() diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5359a732562c4..4b6bb8ce8c56f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -594,15 +594,22 @@ pub(crate) struct Queue { source_readers: Arc, source_hint: crate::g17::sync_input::ReaderHint, /// The doorbell withheld from publications that had [`WITHHOLD_DEPTH`] - 1 or more - /// unretired commands of this queue ahead of them, and the publication of the first. - withheld_doorbell: Option<(u64, u64)>, + /// unretired commands of this queue ahead of them, the publication of the first, and the + /// compute work slots written at its priority once its slot was. + withheld_doorbell: Option<(u64, u64, u64)>, } impl Queue { /// Returns the withheld doorbell once at most one unretired command of this queue - /// precedes the first publication it covers. - pub(crate) fn due_doorbell(&mut self) -> Option { - let (doorbell, first) = self.withheld_doorbell?; + /// precedes the first publication it covers. A doorbell of the same class rung after the + /// last withheld work slot was written announced every one of them: nothing is left to + /// ring. `announced` gives the slots announced per class. + pub(crate) fn due_doorbell(&mut self, announced: impl Fn(u8) -> u64) -> Option { + let (doorbell, first, written) = self.withheld_doorbell?; + if announced(((doorbell >> 2) & 3) as u8) >= written { + self.withheld_doorbell = None; + return None; + } if self.active.iter().filter(|active| active.published < first).count() > 1 { return None; } @@ -1913,6 +1920,8 @@ pub(crate) trait Host { fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; fn notify(&mut self, message: u64) -> Result; + fn notify_compute(&mut self, priority: u8, message: u64) -> Result; + fn compute_written(&self, priority: u8) -> u64; fn note_submission(&mut self) -> Result; } @@ -2260,12 +2269,14 @@ impl Queue { fence(Ordering::SeqCst); let doorbell = (0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2; if self.active.len >= WITHHOLD_DEPTH - && self.withheld_doorbell.is_none_or(|(withheld, _)| withheld == doorbell) + && self.withheld_doorbell.is_none_or(|(withheld, _, _)| withheld == doorbell) { - self.withheld_doorbell.get_or_insert((doorbell, published)); + let written = host.compute_written(policy.priority()); + let entry = self.withheld_doorbell.get_or_insert((doorbell, published, written)); + entry.2 = written; } else { - host.notify(doorbell)?; - if self.withheld_doorbell.is_some_and(|(withheld, _)| withheld == doorbell) { + host.notify_compute(policy.priority(), doorbell)?; + if self.withheld_doorbell.is_some_and(|(withheld, _, _)| withheld == doorbell) { self.withheld_doorbell = None; } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 625339afafeaa..90d980bc427c2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1394,6 +1394,12 @@ impl compute::Host for ComputeHost<'_> { fn notify(&mut self, message: u64) -> Result { self.port.notify(message) } + fn notify_compute(&mut self, priority: u8, message: u64) -> Result { + self.port.notify_compute(priority, message) + } + fn compute_written(&self, priority: u8) -> u64 { + self.port.compute_written(priority) + } fn note_submission(&mut self) -> Result { self.effort.submit(); super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) @@ -1438,6 +1444,12 @@ impl compute::Host for DirectHost<'_> { fn notify(&mut self, message: u64) -> Result { self.port.notify(message) } + fn notify_compute(&mut self, priority: u8, message: u64) -> Result { + self.port.notify_compute(priority, message) + } + fn compute_written(&self, priority: u8) -> u64 { + self.port.compute_written(priority) + } fn note_submission(&mut self) -> Result { Ok(()) } @@ -1680,13 +1692,13 @@ impl super::Firmware { freed |= busy && queue.channel_free(); // Withheld doorbells are rung once their command is next but one; one per // priority class announces every queue's work slots on that ring. - if let Some(doorbell) = queue.due_doorbell() { + if let Some(doorbell) = queue.due_doorbell(|class| port.compute_announced(class)) { let class = ((doorbell >> 2) & 3) as usize; doorbells[class] = doorbell; } } for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { - self.primary.notify(doorbell)?; + self.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. From d38cce2d8be54bdbc01e90e1a92ef3dd65ac0640 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 13:42:20 -0400 Subject: [PATCH 55/64] drm/asahi: announce withheld G17 compute doorbells whenever retirement stalls A withheld compute doorbell waited for the host to retire all but one of the commands ahead of it. Retirement can stall even though the firmware has finished those commands, for example while a completion record waits up to 50 ms for its stamp. It can also leave ordinary operation through a timeout, kill, quarantine, replay, recovery witness or exit. The firmware then sat idle on a command it had never been told about. Count only predecessors that have no completion record yet. Ring a withheld doorbell at once from the compute scan, and from the timeout and cancellation paths, whenever the queue is settling. Never withhold while it is settling. The firmware then knows every published command, as it does without withholding. --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 42 ++++++++++++++++--- .../drm/asahi/g17/runtime/compute_control.rs | 14 ++++++- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 4b6bb8ce8c56f..9382d50d90b79 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -600,22 +600,53 @@ pub(crate) struct Queue { } impl Queue { - /// Returns the withheld doorbell once at most one unretired command of this queue - /// precedes the first publication it covers. A doorbell of the same class rung after the - /// last withheld work slot was written announced every one of them: nothing is left to - /// ring. `announced` gives the slots announced per class. + /// Returns the withheld doorbell once at most one command of this queue that precedes the + /// first publication it covers has no completion record yet. A doorbell of the same class + /// rung after the last withheld work slot was written announced every one of them: nothing + /// is left to ring. `announced` gives the slots announced per class. + /// + /// Retirement waiting for a stamp does not hold the doorbell back, and any departure from + /// ordinary retirement (timeout, kill, quarantine, replay, recovery or exit) rings it at + /// once: the firmware then sees every published command, as without withholding. pub(crate) fn due_doorbell(&mut self, announced: impl Fn(u8) -> u64) -> Option { let (doorbell, first, written) = self.withheld_doorbell?; if announced(((doorbell >> 2) & 3) as u8) >= written { self.withheld_doorbell = None; return None; } - if self.active.iter().filter(|active| active.published < first).count() > 1 { + let settling = self.settling(); + let incomplete = self + .active + .iter() + .filter(|active| active.published < first && active.observed.is_none()) + .count(); + if !settling && incomplete > 1 { return None; } self.withheld_doorbell = None; Some(doorbell) } + + /// The queue has left ordinary retirement: a timeout, kill, quarantine, replay, recovery + /// witness or exit is in progress. + fn settling(&self) -> bool { + self.quarantined + || self.released + || self.exit.is_some() + || self.replays.len != 0 + || self.replay_retirement + || self.awaiting_witness + || self.retire_pending + || self.retired_by_teardown + || self.spared_quarantine + || self.failure_status_pending + || self.terminal.is_some() + } + + /// Takes the withheld doorbell regardless of its predecessors. + pub(crate) fn take_withheld_doorbell(&mut self) -> Option { + self.withheld_doorbell.take().map(|(doorbell, _, _)| doorbell) + } /// The caller reserves a QID first, cancels it on construction failure, and retains every /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( @@ -2269,6 +2300,7 @@ impl Queue { fence(Ordering::SeqCst); let doorbell = (0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2; if self.active.len >= WITHHOLD_DEPTH + && !self.settling() && self.withheld_doorbell.is_none_or(|(withheld, _, _)| withheld == doorbell) { let written = host.compute_written(policy.priority()); diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 4f519743aed53..42282aea62023 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -69,7 +69,14 @@ impl Backend { if !queue.owns_packet(packet) { return Ok(None); } - queue.timeout(&mut |event| deferred.push(event)) + let status = queue.timeout(&mut |event| deferred.push(event)); + // A timed-out queue leaves ordinary retirement: announce what it withheld. + let withheld = queue.take_withheld_doorbell(); + drop(queue); + if let Some(doorbell) = withheld { + firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; + } + status })(); if result.as_ref().is_ok_and(|status| status.is_some()) { // VM status and peer settlement precede signalling this timeout. The backend @@ -92,6 +99,11 @@ impl Backend { firmware.scan_compute(None, &mut deferred)?; if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; + let withheld = queue.take_withheld_doorbell(); + drop(queue); + if let Some(doorbell) = withheld { + firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; + } } Ok(()) })(); From 2d8286101c8631939cb13ce1b7b14c9041856ca6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 16:45:52 -0400 Subject: [PATCH 56/64] drm/asahi: build the G17 driver without warnings Kernels built with CONFIG_WERROR fail on the G17 changes. trace.c defines the eight tracepoint entry points that only Rust calls, so nothing in C declares them (-Wmissing-prototypes). The Rust side also leaves code without users: the compute host's generic doorbell, a recovery poll constant, the old capacity-wait helper and an unused import, and, without G16, the CPU timestamp mapping that only the G16 timestamp path uses. Declare the entry points in trace_api.h, included by trace.c, remove the unused items, and build the CPU timestamp mapping only with G16. Reported-by: RyanTheTide --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 1 - drivers/gpu/drm/asahi/g17/runtime.rs | 13 ------------ drivers/gpu/drm/asahi/g17/trace.c | 2 ++ drivers/gpu/drm/asahi/g17/trace.rs | 2 +- drivers/gpu/drm/asahi/g17/trace_api.h | 24 ++++++++++++++++++++++ drivers/gpu/drm/asahi/mmu.rs | 3 +++ drivers/gpu/drm/asahi/timestamp.rs | 1 + 7 files changed, 31 insertions(+), 15 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/trace_api.h diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 9382d50d90b79..9c4c5c92d049d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1950,7 +1950,6 @@ pub(crate) trait Host { fn publish_qid(&mut self, id: kick::Id) -> Result; fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; - fn notify(&mut self, message: u64) -> Result; fn notify_compute(&mut self, priority: u8, message: u64) -> Result; fn compute_written(&self, priority: u8) -> u64; fn note_submission(&mut self) -> Result; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 90d980bc427c2..1600c55eaf47b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -31,7 +31,6 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; -const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; /// Pending admission keeps exactly retired compute backing available for reuse. @@ -1391,9 +1390,6 @@ impl compute::Host for ComputeHost<'_> { fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } - fn notify(&mut self, message: u64) -> Result { - self.port.notify(message) - } fn notify_compute(&mut self, priority: u8, message: u64) -> Result { self.port.notify_compute(priority, message) } @@ -1441,9 +1437,6 @@ impl compute::Host for DirectHost<'_> { fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } - fn notify(&mut self, message: u64) -> Result { - self.port.notify(message) - } fn notify_compute(&mut self, priority: u8, message: u64) -> Result { self.port.notify_compute(priority, message) } @@ -2396,12 +2389,6 @@ impl Registry { } impl Backend { - fn wait_compute_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { - self.reclaim_capacity(admission, reclaim_backing)?; - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - admission.check() - } - /// Compute admission without a channel: give back what pressure allows, then block until a /// channel is released after `observed` (sampled before the failed check). fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c index ae977c13aab3b..23666be389b96 100644 --- a/drivers/gpu/drm/asahi/g17/trace.c +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -2,6 +2,8 @@ #include #include +#include "trace_api.h" + #define CREATE_TRACE_POINTS #include "trace.h" diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs index 57db5abaccef8..b76b46425d2ce 100644 --- a/drivers/gpu/drm/asahi/g17/trace.rs +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -3,7 +3,7 @@ //! Command lifecycle tracepoints (tracefs `events/asahi_g17`). Each call is a //! static-key test in C while its event is disabled. -use kernel::{bindings, dma_fence::RawDmaFence, prelude::*, sync::Arc}; +use kernel::{bindings, dma_fence::RawDmaFence, sync::Arc}; unsafe extern "C" { fn asahi_g17_trace_publish_enabled() -> bool; diff --git a/drivers/gpu/drm/asahi/g17/trace_api.h b/drivers/gpu/drm/asahi/g17/trace_api.h new file mode 100644 index 0000000000000..d09f529a163f0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace_api.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Entry points of the G17 command lifecycle tracepoints, called from Rust + * (g17/trace.rs). + */ +#ifndef _ASAHI_G17_TRACE_API_H +#define _ASAHI_G17_TRACE_API_H + +#include + +struct dma_fence; + +bool asahi_g17_trace_publish_enabled(void); +void asahi_g17_trace_publish(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + struct dma_fence *fence); +void asahi_g17_trace_complete(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end); +void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind); +bool asahi_g17_trace_doorbell_enabled(void); +void asahi_g17_trace_doorbell(u8 engine, u32 power); +void asahi_g17_trace_message(u8 kind, u64 value); +void asahi_g17_trace_event(u8 phase); + +#endif /* _ASAHI_G17_TRACE_API_H */ diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f5b079e9b5382..d67513d6bde4f 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1255,11 +1255,13 @@ impl TimestampVmap { /// Prepared CPU timestamp access with its own vmap; destruction happens after completion /// signalling. +#[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) struct TimestampMapping { mapping: TimestampVmap, offset: usize, } +#[cfg(CONFIG_DRM_ASAHI_G16)] impl TimestampMapping { /// Prepare one bounded CPU timestamp destination on retained GEM backing. pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { @@ -1312,6 +1314,7 @@ impl KernelMapping { /// use a complete atomic store; byte stores support unaligned destinations without /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { let gem = self.0._gem.as_ref().ok_or(EINVAL)?; TimestampMapping::new(gem, self.timestamp_offset(offset)?) diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 0d365a8d930bb..9f13436b4caf5 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -66,6 +66,7 @@ impl Buffer { } } + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { match self { Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), From 9a0ed807e4d3691a5ea46fce2a2424295f8af6ea Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 16:45:52 -0400 Subject: [PATCH 57/64] drm/asahi: renew G17 job deadlines while the GPU keeps retiring work The scheduler fails a published job two seconds after it started unless the job is a ray-tracing or acceleration-structure build. Under load that timer measures how the GPU is shared rather than whether the job makes progress. Beside four other clients, a 1.5 s compute dispatch took up to 5 s, and lower-priority work waited behind higher-priority work for seconds, while the GPU kept retiring other commands at least every 1.2 s. Those jobs failed with a lost device although nothing had hung. Renew a published job's deadline while any command on the GPU retired in the last five seconds, at most fourteen times (thirty seconds in all). A GPU that retires nothing for five seconds has stopped, and its jobs fail as before. --- drivers/gpu/drm/asahi/g17/job.rs | 9 ++++ drivers/gpu/drm/asahi/g17/recovery.rs | 2 +- drivers/gpu/drm/asahi/g17/recovery/safety.rs | 11 +++++ drivers/gpu/drm/asahi/g17/timeout.rs | 44 ++++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 27e54613965cd..8175c992b8a62 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -307,6 +307,7 @@ struct Timeout { backend: Arc, victim_renewals: u8, execution_renewals: u8, + shared_renewals: u8, packet: Arc, } @@ -331,6 +332,7 @@ impl sched::JobImpl for Job { backend: self.backend.clone(), victim_renewals: 0, execution_renewals: 0, + shared_renewals: 0, packet: self.packet.clone(), } } @@ -344,6 +346,13 @@ impl sched::JobImpl for Job { ) { return true; } + // Other work retiring means the GPU is shared, not hung. + if super::timeout::renew_shared( + job.packet.is_published(), super::recovery::since_retired(super::now_ns()), + &mut job.shared_renewals, + ) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); if deferred { job.victim_renewals = job.victim_renewals.saturating_add(1); diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 84a01d975c36d..36f1ceac9f821 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -10,7 +10,7 @@ mod attribution; mod slot; mod safety; -pub(crate) use safety::note_retired; +pub(crate) use safety::{note_retired, since_retired}; pub(crate) use attribution::Sources; use attribution::Attribution; diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index b97fc26914329..082416f0cb9a6 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -10,16 +10,27 @@ const LIMIT: u8 = 8; /// Commands retired successfully. A watchdog storm means the GPU stopped making progress; /// timeouts of overloaded clients while other work keeps retiring are not one. static RETIRED: AtomicU64 = AtomicU64::new(0); +/// [`crate::g17::now_ns`] of the latest successful retirement, zero before the first. +static LAST_RETIRED_NS: AtomicU64 = AtomicU64::new(0); /// Called for every command that retires without error. pub(crate) fn note_retired() { RETIRED.fetch_add(1, Ordering::Relaxed); + LAST_RETIRED_NS.store(crate::g17::now_ns(), Ordering::Relaxed); } pub(super) fn retired() -> u64 { RETIRED.load(Ordering::Relaxed) } +/// Nanoseconds since any command last retired successfully, or `None` before the first. +pub(crate) fn since_retired(now: u64) -> Option { + match LAST_RETIRED_NS.load(Ordering::Relaxed) { + 0 => None, + last => Some(now.saturating_sub(last)), + } +} + pub(super) struct Storm { start: Option, count: u8, diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index d951bd164d6e9..7b4ef4137b795 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -53,6 +53,32 @@ pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) true } +/// Polls a published job may be renewed while the GPU keeps retiring work. A job +/// shares the GPU's cores with every other runnable queue, so its wall time grows +/// with the other clients' work: beside four other clients a 1.5 s dispatch took up +/// to 5 s, and lower-priority work waits behind higher-priority work. It has not +/// hung while the GPU keeps completing commands. Fourteen renewals bound the wait at +/// thirty seconds, after which a job that still has not finished fails as before. +const SHARED_RENEWALS: u8 = 14; + +/// The GPU counts as live while some command retired this recently. Under heavy +/// sharing the longest gap between retirements measured 1.2 s; a GPU that retires +/// nothing for this long has stopped. +pub(crate) const LIVENESS_NS: u64 = 5_000_000_000; + +/// Renews a published job's deadline while some command retired within +/// [`LIVENESS_NS`] (`since_retired`), at most [`SHARED_RENEWALS`] times. +pub(crate) fn renew_shared(published: bool, since_retired: Option, renewals: &mut u8) -> bool { + if !published + || !since_retired.is_some_and(|since| since < LIVENESS_NS) + || *renewals >= SHARED_RENEWALS + { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. @@ -215,6 +241,24 @@ mod tests { assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); } #[test] + fn a_live_gpu_renews_a_published_job_for_thirty_seconds() { + let mut renewals = 0; + for poll in 1..=14u8 { + assert!(renew_shared(true, Some(4_900_000_000), &mut renewals)); + assert_eq!(renewals, poll); + } + assert!(!renew_shared(true, Some(0), &mut renewals)); + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 30_000); + } + #[test] + fn a_stopped_gpu_or_unpublished_job_keeps_the_deadline() { + let mut renewals = 0; + assert!(!renew_shared(true, Some(LIVENESS_NS), &mut renewals)); + assert!(!renew_shared(true, None, &mut renewals)); + assert!(!renew_shared(false, Some(0), &mut renewals)); + assert_eq!(renewals, 0); + } + #[test] fn exhausted_or_invalid_credit_never_wraps() { for value in 4..=u8::MAX { let mut renewals = value; From afbfbc0b89ca4779b13cefff5bed0b4036751b2c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 16:45:52 -0400 Subject: [PATCH 58/64] drm/asahi: kill closed G17 contexts only while no other work runs A client whose work failed closes its queues, and 250 ms later the driver asks the firmware to kill the closed context's remaining commands. While other contexts' commands are running, that kill makes the firmware restart the GPU and fail everything in flight, and a few repeats crash it. On an otherwise idle GPU the same kill is handled cleanly. Publish the kill only while no other context has a render pair or compute queue in flight, or once the GPU has retired nothing for five seconds, the watchdog's liveness window. Until then the closed context's queues stay quarantined and retained, and its commands retire like any others once the GPU reaches them, as commands with ray traversal scratch already do. Deferred kills are rechecked on every service pass. --- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 26 ++++++++++++++++++- .../drm/asahi/g17/runtime/render/teardown.rs | 8 ++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 69bc7550fc4e9..bf9d023038a51 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -9,6 +9,7 @@ use crate::g17::{ fw::channels::{ContextKill, ControlRecord}, queue::compute::exit, recovery::Memory as _, + timeout::LIVENESS_NS, Firmware, MSG_CONTROL_NOTIFY, }; use core::sync::atomic::{fence, Ordering}; @@ -82,7 +83,7 @@ impl Firmware { }) else { return Ok(()); }; - if self.render_control_backpressured() { + if self.render_control_backpressured() || self.kill_unsafe(context, now) { return Ok(()); } let (owner, target) = { @@ -149,6 +150,29 @@ impl Firmware { Ok(()) } + /// A kill while other contexts' commands run makes the firmware restart the + /// GPU, and repeated ones crash it; on an otherwise idle GPU it is safe. A + /// closed context's commands left alone retire like any others once the GPU + /// reaches them, so its queues stay quarantined and retained until then. Only + /// a GPU that retired nothing for [`LIVENESS_NS`] has stopped, and then the + /// kill proceeds. + fn kill_unsafe(&self, context: &Arc, now: u64) -> bool { + let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); + if stalled { + return false; + } + self.queues.render.foreign_in_flight(context) + || self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + queue.in_flight() && !queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + }) + } + pub(in crate::g17) fn compute_exit_polling(&self, now: u64) -> bool { if self .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index eb04f80361955..e54b1f4638f58 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -46,6 +46,14 @@ impl RenderState { Ok(true) } + /// Some render pair runs another context's commands. + pub(in crate::g17::runtime) fn foreign_in_flight(&self, context: &Arc) -> bool { + self.entries.iter().flatten().any(|entry| { + !entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) + && entry.pair.as_ref().is_some_and(|pair| pair.in_flight()) + }) + } + pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { context.render_pools_released(|id, control| self.pools.released(id, control)) && self From 31279abf0648c00e454b181f01d65b277a32fb90 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 18:22:31 -0400 Subject: [PATCH 59/64] drm/asahi: fail G17 render groups a recovery dropped out of order at once After a firmware restart, the firmware retires every render group it had started on a pair without running it: the groups the recovery table lists, and those whose tiler had run while their fragment stage was still pending. Groups it had not started run as usual. The host republishes dropped groups only when every retained group was dropped, so that none runs after a live successor. Once a later group has run, the dropped ones never qualify, and in-order retirement cannot pass the unexecuted front either. The pair then waited for the render watchdog: about 2 s later before deadline renewal, up to 30 s with it while other work keeps retiring. Its owner then saw ETIMEDOUT and a poisoned VM, even when the client was innocent and none of its work had started. In a stress run with forced recoveries, 10 of 11 recoveries left such a pair. Replay was also wrong for a dropped group that a compute command waited on: the wait ends when firmware retires the group, so the consumer may read output the group never wrote while the replay runs it afterwards. A client in the same stress run read back the previous iteration's image this way without any error. When firmware has retired every group of the pair but a dropped group cannot run again in order (a later group already ran, the dropped group had started, or a compute command waited on it), fail the whole set at once, after the same double observation as a replay. The groups that ran fail too: they consumed output the dropped ones never wrote. The work is innocent, so it fails with ENODATA, which its owner sees, without poisoning the VM. Guilty and blamed sets are left to the timeout as before, and a group that ran ahead of every dropped one is still retired in order. --- drivers/gpu/drm/asahi/g17/queue/render.rs | 79 ++++++++++++++++--- .../asahi/g17/queue/render/dependencies.rs | 26 +++--- drivers/gpu/drm/asahi/g17/replay.rs | 67 +++++++++++++++- .../asahi/g17/runtime/render/dependencies.rs | 3 + 4 files changed, 149 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 9469d526f49c8..696a1486eb0ec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -1333,10 +1333,9 @@ impl Pair { let blamed = active.packet.completion.blamed(); let exhausted = active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; - // decide() stops at the first group that is not abandoned, an error included. - let abandoned = innocent - && !blamed - && (|| -> Result { + // decide() stops at the first group that is still live, an error included. + let (abandoned, ran, lost) = if innocent && !blamed { + (|| -> Result<(bool, bool, bool)> { let mut observer = Observer { memory, qids, @@ -1347,21 +1346,33 @@ impl Pair { let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; // A torn sample is no proof; a later poll or the timeout looks again. - // Interrupted work is run again only if it never started. - Ok(matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, false), - Ok(true) - ) && (!active.packet.completion.interrupted() || snapshot.untouched())) + let witness = |executed| { + matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, executed), + Ok(true) + ) + }; + let unexecuted = witness(false); + // Interrupted work is run again only if it never started, and work that + // another queue waited on never: the wait ended when firmware retired it. + let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) + || active.packet.completion.has_firmware_consumer(); + Ok((unexecuted && !lost, !unexecuted && witness(true), unexecuted && lost)) })() .unwrap_or_else(|error: Error| { failure = Some(error); - false - }); + (false, false, false) + }) + } else { + (false, false, false) + }; Group { innocent, blamed, abandoned, exhausted, + ran, + lost, } }); let decision = replay::decide(groups); @@ -1376,7 +1387,7 @@ impl Pair { front.abandoned_since = None; return Ok(false); } - Decision::Replay => {} + Decision::Replay | Decision::Fail => {} } if grace { let Some(since) = front.abandoned_since else { @@ -1387,6 +1398,9 @@ impl Pair { return Ok(false); } } + if decision == Decision::Fail { + return self.fail_dropped(host, defer); + } // decide() lets only the front be exhausted. An exhausted front fails, and its // successors, which consume its output, fail with it rather than run without it. let mut failing = self.active.front().is_some_and(|front| { @@ -1445,6 +1459,45 @@ impl Pair { Ok(true) } + /// Firmware retired every group of the pair after a restart, but some were dropped + /// unexecuted and cannot run again in order (see `replay::decide`). Every group fails now, + /// the ones that ran too: they consumed output the dropped ones never wrote. The work is + /// innocent, so it fails without poisoning its VM, with ENODATA, which its owner sees, + /// instead of waiting for the render timeout to fail it with ETIMEDOUT. + fn fail_dropped( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + pr_warn!( + "G17: render slot {}: a GPU recovery dropped groups that cannot run again in order; failing {} groups\n", + self.slot, + self.active.len() + ); + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + while let Some(front) = self.active.front_mut() { + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let completion = front.packet.completion.clone(); + completion.classify(recovery::Class::Spared); + defer(Deferred::Retired(completion, Err(ENODATA)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + Ok(true) + } + /// A front group a restart interrupted after its tiler started, which firmware /// then retired without writing its fragment end, may have run partly. It is /// not replayed (blending or storage writes could apply twice) and fails now, @@ -1943,7 +1996,7 @@ impl Pair { pub(crate) fn own_dependencies(&self, owner: u64, packet: &Arc) -> Result { let mut dependencies = RenderDependencies::EMPTY; let mut frontier = crate::g17::dependency::Frontier::new(); - self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier)?; + self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier, false)?; for dependency in frontier.as_slice() { *dependencies.render.get_mut(dependencies.render_count).ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index 53a0fa6e449ed..ab96b48de3e35 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -31,6 +31,7 @@ fn include( prefix: Option, packet: &Arc, word: u64, + compute: bool, ) -> Result { if !Arc::ptr_eq(packet.completion.status(), status) { return Ok(()); @@ -38,12 +39,15 @@ fn include( if !job::Order::contains(prefix, packet.order.sequence) { return Ok(()); } - frontier.include( - prefix, - packet.order.sequence, - job::fence_status(&packet.completion.scheduler_fence()), - word, - ) + let status = job::fence_status(&packet.completion.scheduler_fence()); + frontier.include(prefix, packet.order.sequence, status, word)?; + // A compute consumer's firmware wait ends once firmware retires this group, even unexecuted + // after a restart, so it may then read output the group never wrote. Render consumers of the + // same owner follow the group on its pair, and a replay keeps that order. + if compute && status == 0 { + packet.completion.note_firmware_consumer(); + } + Ok(()) } impl Dependencies { @@ -60,12 +64,13 @@ impl Dependencies { status: &Arc, prefix: Option, frontier: &mut Frontier, + compute: bool, ) -> Result { if self.owner != Some(owner) { return Ok(()); } for entry in self.entries.iter().flatten() { - include(frontier, status, prefix, &entry.packet, entry.word)?; + include(frontier, status, prefix, &entry.packet, entry.word, compute)?; } Ok(()) } @@ -80,7 +85,7 @@ impl Dependencies { return Ok(()); } for entry in self.entries.iter().flatten() { - include(frontier, status, prefix, &entry.packet, entry.ta_word)?; + include(frontier, status, prefix, &entry.packet, entry.ta_word, false)?; } Ok(()) } @@ -112,6 +117,7 @@ impl Pair { status: &Arc, prefix: Option, frontier: &mut Frontier, + compute: bool, ) -> Result { if self.owner != Some(owner) { return Ok(()); @@ -120,7 +126,7 @@ impl Pair { for entry in self.active.iter() { let word = KickDependency::new(qid, entry.kicks[1]).map_or(u64::MAX, KickDependency::word); - include(frontier, status, prefix, &entry.packet, word)?; + include(frontier, status, prefix, &entry.packet, word, compute)?; } Ok(()) } @@ -138,7 +144,7 @@ impl Pair { for entry in self.active.iter() { let word = KickDependency::new(qid, entry.kicks[0]).map_or(u64::MAX, KickDependency::word); - include(frontier, status, prefix, &entry.packet, word)?; + include(frontier, status, prefix, &entry.packet, word, false)?; } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs index 94daa5ec0dfeb..7668871376bad 100644 --- a/drivers/gpu/drm/asahi/g17/replay.rs +++ b/drivers/gpu/drm/asahi/g17/replay.rs @@ -16,6 +16,12 @@ pub(crate) struct Group { pub(crate) abandoned: bool, /// Already republished the maximum number of times. pub(crate) exhausted: bool, + /// Retired by firmware after executing in full. + pub(crate) ran: bool, + /// Retired by firmware unexecuted, but never to run again: a recovery interrupted it after + /// its tiler started, so it may have run partly, or a command of another queue waited on it, + /// and that wait ended when firmware retired the group, before its output existed. + pub(crate) lost: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -25,26 +31,54 @@ pub(crate) enum Decision { Wait, /// Never replay this set; the render timeout settles it. Refuse, + /// Firmware has retired every group, but a dropped group cannot run again in order: a later + /// group already ran, or the dropped group had started or had a consumer. Fail the set now. + Fail, } /// Groups are read in order and the first one that decides stops the walk, so a caller can /// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its /// successors, which consume its output, with it. An exhausted successor would break the order. +/// +/// After a restart the firmware retires the groups it had started on a pair without running +/// them, while groups it had not yet started run as usual. Once such a later group has run, a +/// replay of the dropped ones would run them after it; a group that had started cannot run +/// twice; and a group another queue waited on may already have been read before it ran. None of +/// these is replayed: the whole set fails at once instead of waiting for the render timeout. A +/// group that ran ahead of every dropped one is left to in-order retirement. pub(crate) fn decide(groups: impl Iterator) -> Decision { let mut front_exhausted = None; let mut successor_exhausted = false; + let mut dropped = false; + let mut unordered = false; for group in groups { if !group.innocent || group.blamed { return Decision::Refuse; } + if group.ran { + if !dropped { + return Decision::Wait; + } + unordered = true; + continue; + } + if group.lost { + dropped = true; + unordered = true; + continue; + } if !group.abandoned { return Decision::Wait; } + dropped = true; match front_exhausted { None => front_exhausted = Some(group.exhausted), Some(_) => successor_exhausted |= group.exhausted, } } + if unordered { + return Decision::Fail; + } match front_exhausted { Some(true) => Decision::Replay, Some(false) if !successor_exhausted => Decision::Replay, @@ -56,10 +90,13 @@ pub(crate) fn decide(groups: impl Iterator) -> Decision { mod tests { use super::*; fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { - Group { innocent, blamed: false, abandoned, exhausted } + Group { innocent, blamed: false, abandoned, exhausted, ran: false, lost: false } } - const OK: Group = Group { innocent: true, blamed: false, abandoned: true, exhausted: false }; - const BLAMED: Group = Group { innocent: true, blamed: true, abandoned: true, exhausted: false }; + const OK: Group = + Group { innocent: true, blamed: false, abandoned: true, exhausted: false, ran: false, lost: false }; + const BLAMED: Group = Group { blamed: true, ..OK }; + const RAN: Group = Group { abandoned: false, ran: true, ..OK }; + const LOST: Group = Group { abandoned: false, lost: true, ..OK }; #[test] fn every_group_abandoned_and_innocent_is_replayed() { assert_eq!(decide([OK].into_iter()), Decision::Replay); @@ -106,4 +143,28 @@ mod tests { Decision::Replay ); } + #[test] + fn a_group_that_ran_after_dropped_ones_fails_the_set() { + // Seen after restarts: the started groups were dropped, the queued ones ran. + assert_eq!(decide([OK, OK, RAN, RAN].into_iter()), Decision::Fail); + assert_eq!(decide([OK, RAN, OK].into_iter()), Decision::Fail); + // Also with an exhausted front, which a replay would only fail later. + assert_eq!(decide([group(true, true, true), RAN].into_iter()), Decision::Fail); + } + #[test] + fn a_group_that_ran_first_is_left_to_retirement() { + assert_eq!(decide([RAN].into_iter()), Decision::Wait); + assert_eq!(decide([RAN, OK].into_iter()), Decision::Wait); + } + #[test] + fn a_started_dropped_group_fails_the_set() { + assert_eq!(decide([OK, LOST].into_iter()), Decision::Fail); + assert_eq!(decide([LOST, OK].into_iter()), Decision::Fail); + } + #[test] + fn failing_still_waits_for_live_groups_and_refuses_guilty_ones() { + assert_eq!(decide([OK, RAN, group(true, false, false)].into_iter()), Decision::Wait); + assert_eq!(decide([OK, RAN, group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([LOST, BLAMED].into_iter()), Decision::Refuse); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index 2b14b4548e0db..e150a10e2c9bc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -12,6 +12,7 @@ impl Registry { packet: &Arc, ) -> Result { let mut frontier = Frontier::new(); + let compute = packet.engine() == crate::g17::job::Engine::Compute; for entry in self.render.entries.iter().flatten() { if let Some(pair) = entry.pair.as_ref() { pair.collect_dependencies( @@ -19,6 +20,7 @@ impl Registry { packet.completion.status(), packet.order.wait_through[0], &mut frontier, + compute, )?; } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { dependencies.collect( @@ -26,6 +28,7 @@ impl Registry { packet.completion.status(), packet.order.wait_through[0], &mut frontier, + compute, )?; } } From 7012a59ed129baf969d83a3aa4d90f2a6cb410f2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 18:22:31 -0400 Subject: [PATCH 60/64] drm/asahi: announce withheld G17 compute doorbells after a recovery A compute doorbell is withheld while two or more unretired commands of its queue precede it, and any departure from ordinary retirement is meant to ring it at once. A recovery that touches no command of the queue left it withheld until those predecessors retired. Once the firmware has resumed from a restart, ring every withheld doorbell, one per priority class, before any replay is published. --- drivers/gpu/drm/asahi/g17/recovery.rs | 4 ++++ drivers/gpu/drm/asahi/g17/runtime/events.rs | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 36f1ceac9f821..3bb9d20190b57 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -107,6 +107,9 @@ pub(super) trait Queues { fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; fn release_retired_compute(&mut self); + /// Ring every compute doorbell still withheld: a restart is a departure from ordinary + /// retirement, after which the firmware must see every published command. + fn announce_withheld(&mut self); } /// Live shared-memory operations. The handshake word uses aligned 32-bit accesses; transitions @@ -296,6 +299,7 @@ pub(super) fn service( } let scan_error = host.rescan_compute().err(); host.release_retired_compute(); + host.announce_withheld(); Ok(ServiceResult { scan_error, target_error }) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index e13ec3e35e13c..4cf2dd0fafb97 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -236,6 +236,26 @@ impl recovery::Queues for Service<'_> { let result = self.firmware.release_compute_witnesses(self.deferred); self.settlement_error = self.settlement_error.or(result.err()); } + fn announce_withheld(&mut self) { + let mut doorbells = [0u64; 4]; + for mut queue in self + .firmware + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if let Some(doorbell) = queue.take_withheld_doorbell() { + doorbells[((doorbell >> 2) & 3) as usize] = doorbell; + } + } + // One doorbell per priority class announces every queue's work slots on that ring. + for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { + let result = self.firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell); + self.settlement_error = self.settlement_error.or(result.err()); + } + } } impl Firmware { From 8e257af33a6c4036ee56bc8a628e733dba2bd525 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 19:17:20 -0400 Subject: [PATCH 61/64] drm/asahi: move the G17 cumulative-signals feature to bit 29 Feature bit 17 is also used for render resources by the M5 userspace line, whose features take bits 17 to 28. Advertise cumulative signals as bit 29, which no line uses, so one Mesa can serve both kernels. A kernel and Mesa that disagree on the bit fall back to ordinary per-point signals, so either mix still works. --- drivers/gpu/drm/asahi/g17/sync_output.rs | 2 +- include/uapi/drm/asahi_drm.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/sync_output.rs b/drivers/gpu/drm/asahi/g17/sync_output.rs index 50585befb3b71..928ebe2319394 100644 --- a/drivers/gpu/drm/asahi/g17/sync_output.rs +++ b/drivers/gpu/drm/asahi/g17/sync_output.rs @@ -6,7 +6,7 @@ use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; // Wire values also documented in include/uapi/drm/asahi_drm.h. Keeping these // local permits building this module against an older kernel's Rust bindings. pub(crate) const SYNC_CUMULATIVE: u32 = 2; -pub(crate) const FEATURE_CUMULATIVE_SIGNALS: u64 = 1 << 17; +pub(crate) const FEATURE_CUMULATIVE_SIGNALS: u64 = 1 << 29; unsafe extern "C" { fn asahi_g17_signals_new(syncs: *const SignalSpec, count: usize, anchor: usize, diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 836739ffbbebf..66d5188199383 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -374,7 +374,7 @@ enum drm_asahi_feature { * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs * include the earlier points of that timeline without adding GPU waits. */ - DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 17, + DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 From 76d4e880149e5592396001f728f7803efb616377 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 20:29:35 -0400 Subject: [PATCH 62/64] drm/asahi: build the G17 compute QID budget test against the variant constructor The test that grows compute QIDs with free identities still called Registry::new() with the clusters and descriptor flags only, from before the constructor took the deflake setting and the firmware variant, so a KUnit build of the driver failed. Construct the registry as the other tests do, for one cluster on the 26A428 firmware. --- drivers/gpu/drm/asahi/g17/runtime.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 1600c55eaf47b..c5675cbec5f79 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -2547,7 +2547,7 @@ mod shared_capacity_tests { #[test] fn compute_grows_with_free_qids_and_narrows_queue_admission() { - let mut registry = KBox::init(Registry::new(1, [0; 2]), GFP_KERNEL).unwrap(); + let mut registry = KBox::init(Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), GFP_KERNEL).unwrap(); let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); for _ in 0..20 { slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); From d0da7e79dc9c99020d109e95d18256cd5b3d6bc2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 20:35:37 -0400 Subject: [PATCH 63/64] drm/asahi: check the G17 QID floor test against the open-queue compute budget The test that a firmware QID floor admits only the render pairs it leaves reserved every compute channel before opening any logical queue. Compute QIDs are now sized by the open logical queues, so on 25G83's floor that first reservation was refused and the test panicked. Open the 42 logical queues the floor allows without the up-front compute reservations, and check as before that the next one is refused when it is created. --- drivers/gpu/drm/asahi/g17/runtime.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index c5675cbec5f79..e04295d576606 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -2508,9 +2508,6 @@ mod shared_capacity_tests { GFP_KERNEL, ) .unwrap(); - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { - registry.reserve_compute(owner).unwrap(); - } let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); for owner in 1..=u64::from(limit) { From 18b408b7368b9470c22d92c9afe1991d0fe608ba Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 25 Oct 2026 20:58:53 -0400 Subject: [PATCH 64/64] drm/asahi: check the G17 admission churn test against its deadline, not the clock The test that gate churn cannot renew a compute admission budget looped until the budget ran out and then required the whole loop to have taken less than 500 ms. The loop does not sleep, so that bound only measured how long the test thread was descheduled, and a KUnit run under KVM on a shared CI host exceeded it. Check the property instead: every admission granted came from an attempt that started before the deadline, and the first attempt after it is refused with EBUSY. --- drivers/gpu/drm/asahi/g17/preparation.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 3cafecf5d8206..eb2366fa3451b 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -361,9 +361,9 @@ mod compute_deadline_tests { #[test] fn recovery_and_binding_retry_churn_cannot_renew_budget() { let gate = Gate::new().unwrap(); - let start = super::super::now_ns(); + let deadline = super::super::now_ns() + 20_000_000; let check = || Ok(()); - let admission = Admission::new(start + 20_000_000, &check); + let admission = Admission::new(deadline, &check); loop { // Re-entry models a channel taken between admission/publication. // Recovery joins each dropped lease, then wakes the same budget. @@ -371,12 +371,17 @@ mod compute_deadline_tests { gate.changed.notify_all(); gate.wait_drained(); assert!(gate.reopen()); + // However often the gate reopened, an attempt that starts at the deadline + // is refused. Timing the whole loop would also count time descheduled. + let attempted = super::super::now_ns(); match gate.enter_until(&admission) { - Ok(lease) => drop(lease), + Ok(lease) => { + assert!(attempted < deadline); + drop(lease); + } Err(error) => { assert_eq!(error, EBUSY); break; } } } - assert!(super::super::now_ns() - start < 500_000_000); assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); }