diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c index 5e10e8df372f1..64abdcc430580 100644 --- a/drivers/dma-buf/dma-fence-array.c +++ b/drivers/dma-buf/dma-fence-array.c @@ -107,17 +107,11 @@ static bool dma_fence_array_signaled(struct dma_fence *fence) unsigned int i; /* - * We need to read num_pending before checking the enable_signal bit - * to avoid racing with the enable_signaling() implementation, which - * might decrement the counter, and cause a partial check. - * atomic_read_acquire() pairs with atomic_dec_and_test() in - * dma_fence_array_enable_signaling() - * - * The !--num_pending check is here to account for the any_signaled case - * if we race with enable_signaling(), that means the !num_pending check - * in the is_signalling_enabled branch might be outdated (num_pending - * might have been decremented), but that's fine. The user will get the - * right value when testing again later. + * Read the counter before the enable bit so callback installation + * cannot make a partial count look complete. The acquire pairs with + * the counter decrements in dma_fence_array_enable_signaling(). + * If enabling races with the lazy scan, every child must still be + * observed complete before that scan can return true. */ num_pending = atomic_read_acquire(&array->num_pending); if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &array->base.flags)) { @@ -126,8 +120,16 @@ static bool dma_fence_array_signaled(struct dma_fence *fence) return false; } + /* + * Arrays require every member to complete. A pending member already + * proves the array incomplete; polling its successors only repeats work + * during timeline-chain walks. Keep the completion counter and the + * enabled-signaling path unchanged. + */ for (i = 0; i < array->num_fences; ++i) { - if (dma_fence_is_signaled(array->fences[i]) && !--num_pending) + if (!dma_fence_is_signaled(array->fences[i])) + return false; + if (!--num_pending) goto signal; } return false; @@ -256,8 +258,7 @@ EXPORT_SYMBOL(dma_fence_array_init); * and fill it with the fences it wants to add to the object. Ownership of this * array is taken and dma_fence_put() is used on each fence on release. * - * If @signal_on_any is true the fence array signals if any fence in the array - * signals, otherwise it signals when all fences in the array signal. + * The array signals when every fence in the array signals. */ struct dma_fence_array *dma_fence_array_create(int num_fences, struct dma_fence **fences, diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c index 4d9d313b460cb..17761f8de04a7 100644 --- a/drivers/dma-buf/st-dma-fence-unwrap.c +++ b/drivers/dma-buf/st-dma-fence-unwrap.c @@ -15,6 +15,7 @@ struct mock_fence { struct dma_fence base; spinlock_t lock; + atomic_t polls; }; static const char *mock_name(struct dma_fence *f) @@ -22,9 +23,18 @@ static const char *mock_name(struct dma_fence *f) return "mock"; } +static bool mock_signaled(struct dma_fence *fence) +{ + struct mock_fence *f = container_of(fence, struct mock_fence, base); + + atomic_inc(&f->polls); + return false; +} + static const struct dma_fence_ops mock_ops = { .get_driver_name = mock_name, .get_timeline_name = mock_name, + .signaled = mock_signaled, }; static struct dma_fence *__mock_fence(u64 context, u64 seqno) @@ -36,6 +46,7 @@ static struct dma_fence *__mock_fence(u64 context, u64 seqno) return NULL; spin_lock_init(&f->lock); + atomic_set(&f->polls, 0); dma_fence_init(&f->base, &mock_ops, &f->lock, context, seqno); return &f->base; @@ -584,7 +595,71 @@ static void test_unwrap_merge_complex_seqno(struct kunit *test) dma_fence_put(f1); } +/* + * A pending all-of array needs no polls of later members. Keep its + * independent member fences so their polling and out-of-order completion + * remain observable without enabling callbacks on the array. + */ +static void test_array_pending_prefix(struct kunit *test) +{ + struct dma_fence *first = mock_fence(), *last = mock_fence(); + struct dma_fence *array; + struct mock_fence *last_mock; + + if (!first || !last) { + KUNIT_FAIL(test, "Could not allocate member fences"); + goto out; + } + last_mock = container_of(last, struct mock_fence, base); + array = mock_array(2, dma_fence_get(first), dma_fence_get(last)); + if (!array) { + KUNIT_FAIL(test, "Could not allocate fence array"); + goto out; + } + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + KUNIT_EXPECT_EQ(test, atomic_read(&last_mock->polls), 0); + dma_fence_signal(last); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + dma_fence_signal(first); + KUNIT_EXPECT_TRUE(test, dma_fence_is_signaled(array)); + dma_fence_put(array); +out: + dma_fence_put(last); + dma_fence_put(first); +} + +static void test_array_pending_suffix(struct kunit *test) +{ + struct dma_fence *first = mock_fence(), *last = mock_fence(); + struct dma_fence *array; + + if (!first || !last) { + KUNIT_FAIL(test, "Could not allocate member fences"); + goto out; + } + array = mock_array(2, dma_fence_get(first), dma_fence_get(last)); + if (!array) { + KUNIT_FAIL(test, "Could not allocate fence array"); + goto out; + } + dma_fence_signal(first); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + /* Installing callbacks after a lazy partial poll must still account + * for the already signaled first member and wait for the last. + */ + dma_fence_enable_signaling(array); + KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array)); + dma_fence_signal(last); + KUNIT_EXPECT_GT(test, dma_fence_wait_timeout(array, false, HZ), 0L); + dma_fence_put(array); +out: + dma_fence_put(last); + dma_fence_put(first); +} + static struct kunit_case dma_fence_unwrap_cases[] = { + KUNIT_CASE(test_array_pending_prefix), + KUNIT_CASE(test_array_pending_suffix), KUNIT_CASE(test_sanitycheck), KUNIT_CASE(test_unwrap_array), KUNIT_CASE(test_unwrap_chain), diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index 0523740332290..db46090abd943 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DRM_ASAHI) += asahi.o -asahi-y := asahi_core.o fwid.o sysfs.o +asahi-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o +CFLAGS_g17/trace.o := -I$(src)/g17 asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index a2b17082494cb..01598b691ded3 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -127,10 +127,18 @@ pub(crate) struct SyncItem { pub(crate) fence: Option, pub(crate) chain_fence: Option, pub(crate) timeline_value: u64, + pub(crate) cumulative: Option, + pub(crate) timeline: bool, } impl SyncItem { - fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + fn parse_one(file: &DrmFile, mut data: uapi::drm_asahi_sync, out: bool, + bundled: bool) -> Result + { + if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !out || !bundled || data.timeline_value == 0 { return Err(EINVAL); } + data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ; + } match data.sync_type { uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { if data.timeline_value != 0 { @@ -151,6 +159,8 @@ impl SyncItem { syncobj, chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: false, }) } uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { @@ -173,12 +183,10 @@ impl SyncItem { Ok(SyncItem { fence, syncobj, - chain_fence: if out { - Some(dma_fence::FenceChain::new()?) - } else { - None - }, + chain_fence: None, timeline_value: data.timeline_value, + cumulative: None, + timeline: true, }) } _ => { @@ -193,12 +201,14 @@ impl SyncItem { ptr: u64, in_count: u32, out_count: u32, + allow_cumulative: bool, ) -> Result> { let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let mut anchor = None; // SAFETY: We only read this once, so there are no TOCTOU issues. let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); @@ -214,7 +224,25 @@ impl SyncItem { // SAFETY: All bit patterns in the struct are valid let sync = unsafe { sync.assume_init() }; - vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE { + if !allow_cumulative || i < in_count || anchor.is_some() { + return Err(EINVAL); + } + anchor = Some(i as usize); + } + vec.push(Self::parse_one(file, sync, i >= in_count, + allow_cumulative)?, GFP_KERNEL)?; + } + if let Some(anchor) = anchor { + let bundle = crate::g17::sync_output::SignalBundle::new( + &vec[in_count as usize..], anchor - in_count as usize)?; + vec[anchor].cumulative = Some(bundle); + } else { + for sync in &mut vec[in_count as usize..] { + if sync.timeline { + sync.chain_fence = Some(dma_fence::FenceChain::new()?); + } + } } Ok(vec) @@ -1398,7 +1426,8 @@ impl File { copy_commands(&mut vec)?; } let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count, + gpu.as_any().is::())?; mod_dev_dbg!( device, diff --git a/drivers/gpu/drm/asahi/g17/availability.rs b/drivers/gpu/drm/asahi/g17/availability.rs new file mode 100644 index 0000000000000..8014266f0b0ab --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/availability.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Resource-specific scheduler waits. A caller samples a domain before a failed try, then +//! registers against that sample after rolling back unpublished claims; the release that frees +//! the resource bumps the generation and signals exactly the waiters of its own domain. +//! Successful acquisition never allocates a fence, and a release without waiters costs one +//! lock round trip. + +use super::fence::{self, CompletionFence}; +use kernel::{ + c_str, + dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; + +struct Entry { + fence: UserFence, + next: Option>, +} + +struct State { + generation: u64, + waiters: Option>, +} + +#[pin_data] +pub(crate) struct Availability { + #[pin] + state: Mutex, + fences: FenceContexts, +} + +impl Availability { + pub(crate) fn new() -> Result> { + Arc::pin_init( + try_pin_init!(Self { + state <- new_mutex!( + State { generation: 0, waiters: None }, + "g17::Availability" + ), + fences: FenceContexts::new(1, c_str!("asahi_resource"), kernel::static_lock_class!())?, + }), + GFP_KERNEL, + ) + } + + /// Sample before checking capacity under the resource's own synchronization. + pub(crate) fn observe(&self) -> u64 { + self.state.lock().generation + } + + pub(crate) fn has_waiters(&self) -> bool { + self.state.lock().waiters.is_some() + } + + /// A fence the next notification signals. None means capacity changed after the sample: + /// retry instead of waiting. Called with no resource lock held. The recheck after allocation + /// closes both the failed-check/registration and the fence-allocation/release races. A + /// waiter that goes away leaves its entry for the next notification to signal. + pub(crate) fn wait(&self, observed: u64) -> Result> { + if self.observe() != observed { + return Ok(None); + } + let fence = fence::independent(&self.fences)?; + let mut entry = KBox::new( + Entry { fence: fence.clone(), next: None }, + GFP_KERNEL, + )?; + let mut state = self.state.lock(); + if state.generation != observed { + return Ok(None); + } + entry.next = state.waiters.take(); + state.waiters = Some(entry); + Ok(Some(Fence::from_fence(&fence))) + } + + /// Blocks until a notification after `observed`, for at most `timeout_ms`. For callers + /// outside the scheduler that own no other wait. + pub(crate) fn block(&self, observed: u64, timeout_ms: u32, interruptible: bool) -> Result { + let Some(fence) = self.wait(observed)? else { return Ok(()); }; + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + interruptible, + kernel::time::msecs_to_jiffies(timeout_ms) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + Ok(()) + } + + /// Wakes every waiter of this domain. Call after dropping the resource's own lock; fence + /// callbacks run outside the domain lock. + pub(crate) fn notify(&self) { + let mut waiters = { + let mut state = self.state.lock(); + state.generation = state.generation.wrapping_add(1); + state.waiters.take() + }; + while let Some(mut entry) = waiters { + waiters = entry.next.take(); + entry.fence.signal(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/capacity.rs b/drivers/gpu/drm/asahi/g17/capacity.rs index bcbbb2e7e875d..57406e5487a37 100644 --- a/drivers/gpu/drm/asahi/g17/capacity.rs +++ b/drivers/gpu/drm/asahi/g17/capacity.rs @@ -55,6 +55,11 @@ impl kernel::debugfs::Writer for View { let state = self.0.state.lock(); let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); }; writeln!(f, "online=1")?; + let (converted, published) = super::sync_input::counters(); + writeln!(f, "fw_inputs converted={} published={} native={} pending={}", + converted, published, super::sync_input::native_count(), + super::sync_input::pending_count())?; + writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?; firmware.queues.write_capacity(f)?; firmware.pool.write_capacity(f)?; writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?; diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 8e4740b76a166..d96924cbbf9f9 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -138,6 +138,7 @@ impl Rings { record.opcode().asserts_gpu_power(), next, )?; + super::trace::message(super::trace::CONTROL, record.opcode() as u64); Ok(next) } } diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 4f5b87bc853dc..02e7bbc5c33dc 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -27,9 +27,9 @@ mod publication; pub(crate) use publication::{Prepared as PreparedPublication, Publication}; struct Destination { - // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, - cpu: mmu::TimestampMapping, + // The buffer keeps its GPU alias and its CPU mapping alive. + buffer: Arc, + offset: usize, } impl Destination { @@ -45,13 +45,13 @@ impl Destination { .get(timestamp.handle.try_into()?) .ok_or(ENOENT)? .clone(); - let file::Object::TimestampBuffer(mapping) = object; + let file::Object::TimestampBuffer(buffer) = object; let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { + if offset.checked_add(size_of::()).ok_or(EINVAL)? > buffer.size() { return Err(ERANGE); } - let cpu = mapping.timestamp_mapping(offset)?; - Ok(Some(Self { _mapping: mapping, cpu })) + let offset = buffer.timestamp_offset(offset)?; + Ok(Some(Self { buffer, offset })) } fn write(&self, ticks: u64) { @@ -61,13 +61,14 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.cpu.write(ns); + self.buffer.write_timestamp(self.offset, ns); barrier(Ordering::Release); } } -/// Timestamp mappings resolved and retained before any command is enqueued. -pub(crate) struct Destinations(Option; 4]>>); +/// Timestamp destinations resolved and retained before any command is enqueued. Dropping +/// one releases only a buffer reference; the buffer defers its own mapping release. +pub(crate) struct Destinations([Option; 4]); impl Destinations { pub(crate) fn resolve( @@ -82,20 +83,13 @@ impl Destinations { destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; } - // Empty submissions need no cleanup allocation. Timestamped commands - // prepare the unique work item before any output fence is published. - let retained = if destinations.iter().any(Option::is_some) { - Some(crate::cleanup::Deferred::new(destinations)?) - } else { - None - }; - Ok(Self(retained)) + Ok(Self(destinations)) } + pub(crate) fn has_outputs(&self) -> bool { self.0.iter().any(Option::is_some) } + fn iter(&self) -> core::slice::Iter<'_, Option> { - let destinations: &[Option] = - self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); - destinations.iter() + self.0.iter() } } @@ -254,6 +248,7 @@ pub(crate) struct Completion { terminal: Option>, compute: Option, feed: Option>, + sources: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -267,6 +262,9 @@ pub(crate) struct Completion { /// A recovery blamed this command. Unlike `class`, which a later /// classification can change, this never clears. blamed: AtomicBool, + /// A firmware restart listed this command as in flight. Until a replay + /// republishes it from the start, it never reports success. + interrupted: AtomicBool, firmware_consumer: AtomicBool, class: AtomicU8, } @@ -286,6 +284,7 @@ impl Completion { compute: bool, host_predecessors: bool, feed: Option>, + sources: Option>, ) -> Result> { let fence = fence::independent(contexts)?; let ordered = !compute @@ -313,6 +312,7 @@ impl Completion { terminal, compute, feed, + sources, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -322,6 +322,7 @@ impl Completion { replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), blamed: AtomicBool::new(false), + interrupted: AtomicBool::new(false), firmware_consumer: AtomicBool::new(false), class: AtomicU8::new(0), }), @@ -357,6 +358,38 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } + pub(super) fn sources(&self) -> Option<&Arc> { + self.sources.as_ref() + } + pub(super) fn is_input_source(&self, source: &Arc) -> bool { + self.output.output().member.is_source(source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { + self.output.output().member.set_source_ready(ready); + } + pub(super) fn needs_source_ready(&self) -> bool { + self.output.output().member.needs_source_ready() + } + pub(super) fn note_input_error(&self, error: Error) -> bool { + self.output.output().member.note_error(error) + } + /// Called under the device mutex as a successful retirement is queued. + pub(super) fn note_source_retired(&self) { + if self.interrupted() { + // complete() fails it; consumers must not take its output as valid. + self.note_input_error(ECANCELED); + return; + } + match self.check_source_inputs() { + Ok(()) => self.output.output().member.note_retired(), + Err(error) => { self.note_input_error(error); } + } + } + pub(super) fn check_source_inputs(&self) -> Result { + if let Some(sources) = &self.sources { sources.check()?; } + Ok(()) + } + pub(crate) fn work_state(&self) -> Result> { self.output.output().member.work_state() } @@ -397,6 +430,9 @@ impl Completion { .is_ok(); if accepted { self.replay_timeout.store(true, Ordering::Release); + // Callers replay only work proven not to have started; the new + // publication runs from the beginning and reports its own result. + self.interrupted.store(false, Ordering::Release); } accepted } @@ -485,6 +521,35 @@ impl Completion { self.blamed.load(Ordering::Acquire) } + /// A firmware restart listed this command in its recovery table: it was in + /// flight, so it may have run partly. Its stamps and completion record may + /// still look like success. Unless a replay republishes it from the start, + /// it fails and the failure is always reported to the client. + pub(crate) fn note_interrupted(&self) { + self.interrupted.store(true, Ordering::Release); + } + + pub(crate) fn interrupted(&self) -> bool { + self.interrupted.load(Ordering::Acquire) + } + + /// Whether this failure makes the VM unusable. Spared work, and interrupted + /// work no recovery found guilty, fail without poisoning it. + pub(crate) fn poisons(&self, error: Error) -> bool { + let interrupted = self.interrupted(); + (error != ECANCELED || interrupted) && !self.spared() && !(interrupted && !self.guilty()) + } + + /// ECANCELED stays silent only for work that was never interrupted (closed + /// or cancelled by its owner); every other failure reaches the client. + fn report(&self, error: Error) { + if self.poisons(error) { + self.status.record(error); + } else if error != ECANCELED || self.interrupted() { + self.status.report_failure(error); + } + } + /// Installed before the first firmware-visible prefix. Cancellation takes /// the same lock after claiming the signal, so it cannot miss a late lease. pub(crate) fn attach_render_lease( @@ -515,6 +580,13 @@ impl Completion { .map(super::queue::admission::RenderLease::slot) } + /// Called under the device mutex once the render's kicks are committed. + pub(crate) fn note_render_publication(&self, fragment: super::fw::kick::KickDependency) { + if let Some(lease) = self.render.lock().as_ref() { + lease.note_publication(fragment); + } + } + /// Called under the device mutex before deferring the error signal. pub(crate) fn quarantine_render(&self) { if let Some(lease) = self.render.lock().as_ref() { @@ -525,6 +597,7 @@ impl Completion { /// An exact whole-pair drain or processor stop allows rehabilitation. The /// VM pin goes first; the still-held lease excludes reuse until both settle. pub(crate) fn release_after_retirement(&self) { + if let Some(sources) = &self.sources { sources.release(); } drop(self.vm_job.lock().take()); let lease = self.render.lock().take(); if let Some(lease) = lease.as_ref() { @@ -534,6 +607,7 @@ impl Completion { } fn release_healthy(&self) { + if let Some(sources) = &self.sources { sources.release(); } drop(self.render.lock().take()); drop(self.vm_job.lock().take()); } @@ -541,22 +615,28 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. #[track_caller] - pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + pub(crate) fn complete(&self, mut result: Result<[u64; 4]>) { + if result.is_ok() { + if let Err(error) = self.check_source_inputs() { result = Err(error); } + } if self.signalled.swap(true, Ordering::AcqRel) { return; } + if result.is_ok() && self.interrupted() { + result = Err(ECANCELED); + } if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { - feed.note_render_pass(ticks[2], ticks[3]); + // Compute reports its dispatch envelope in the first pair, a + // render its fragment pass in the second. + if self.compute.is_some() { + feed.note_pass(ticks[0], ticks[1]); + } else { + feed.note_pass(ticks[2], ticks[3]); + } } if let Err(error) = result { // Spared work cancelled after another VM's recovery is not this client's device loss. - if self.spared() { - if error != ECANCELED { - self.status.report_failure(error); - } - } else if error != ECANCELED { - self.status.record(error); - } + self.report(error); } self.release_healthy(); self.signal_terminal(result); @@ -590,8 +670,8 @@ impl Completion { } if self.spared() { self.status.report_failure(error); - } else if error != ECANCELED { - self.status.record(error); + } else { + self.report(error); } self.signal_terminal(Err(error)); self.output.finish(Err(error)); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index c93ab25d55ce3..0d3948a2b69ff 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -114,9 +114,16 @@ pub(crate) struct Context { scheduler: Arc, scheduler_generation: u32, published: AtomicBool, + /// Advisory only: publication still checks the physical binding under the + /// device mutex. False skips an inline attempt known to need admission. + compute_binding_hint: AtomicBool, + /// Mixed-engine contexts keep the original inline publication path. + render_seen: AtomicBool, qos: QosLease, owner_pid: u32, states: WorkStates, + /// Signalled when a work state returns; publishers that found none wait on it. + state_returns: Arc, work: Arc>, work_base: u64, render: Arc>>>, @@ -135,6 +142,7 @@ impl Context { execution: Arc, owner_pid: u32, policy: Policy, + state_returns: Arc, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -189,9 +197,12 @@ impl Context { scheduler, scheduler_generation, published: AtomicBool::new(false), + compute_binding_hint: AtomicBool::new(false), + render_seen: AtomicBool::new(false), qos, owner_pid, states: WorkStates::new(), + state_returns, work, work_base, render, @@ -203,6 +214,22 @@ impl Context { pub(crate) fn vm(&self) -> &Vm { self.execution.vm() } + pub(crate) fn compute_binding_hint(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + || self.compute_binding_hint.load(Ordering::Relaxed) + } + /// This logical queue has submitted render work, so its compute may have a render prefix. + pub(crate) fn render_seen(&self) -> bool { + self.render_seen.load(Ordering::Acquire) + } + pub(crate) fn note_render(&self) { + if !self.render_seen.load(Ordering::Relaxed) { + self.render_seen.store(true, Ordering::Release); + } + } + pub(crate) fn note_compute_binding(&self, owned: bool) { + self.compute_binding_hint.store(owned, Ordering::Relaxed); + } pub(crate) fn compute_timestamps(&self) -> &Arc { &self.compute_timestamps } @@ -335,6 +362,7 @@ pub(crate) struct WorkStateLease { word: AtomicU32, } impl WorkStateLease { + pub(super) fn status(&self) -> &Arc { self.context.status() } pub(crate) fn deferred(context: Arc) -> Self { Self { context, @@ -354,8 +382,9 @@ impl WorkStateLease { Err(EBUSY) }; }; - if let Err(error) = self.context.work.lock().initialize(candidate) { - self.context.states.release(candidate); + let initialized = self.context.work.lock().initialize(candidate); + if let Err(error) = initialized { + self.context.release_state(candidate); return Err(error); } match self @@ -364,11 +393,15 @@ impl WorkStateLease { { Ok(_) => Ok(candidate), Err(installed) => { - self.context.states.release(candidate); + self.context.release_state(candidate); Ok(installed) } } } + /// The domain a publisher samples before acquire() and waits on after EBUSY. + pub(crate) fn returns(&self) -> &Arc { + &self.context.state_returns + } pub(crate) fn word(&self) -> u32 { self.word.load(Ordering::Acquire) } @@ -409,7 +442,17 @@ impl Drop for WorkStateLease { fn drop(&mut self) { let word = self.word(); if word != FREE { - self.context.states.release(word); + self.context.release_state(word); + } + } +} + +impl Context { + /// Every return bumps the domain: a publisher that sampled it before its + /// failed acquire() then never sleeps through this node. + fn release_state(&self, word: u32) { + if self.states.release(word) { + self.state_returns.notify(); } } } diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index 0045dff413922..760f352046192 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -410,7 +410,7 @@ impl Feed { Monotonic::ktime_get() as u64 } - pub(crate) fn note_render_pass(&self, start: u64, end: u64) { + pub(crate) fn note_pass(&self, start: u64, end: u64) { if cfg!(CONFIG_CPU_FREQ) { self.detector.note(start, end, Self::now()); } diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 3414e36db9d36..8245e8bca71e6 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -7,10 +7,10 @@ use super::context::{Context, WorkStateLease}; use crate::file::SyncItem; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, AtomicUsize, Ordering}; use kernel::{ bindings, c_str, - dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UniqueFence, UserFence}, new_mutex, prelude::*, str::CStr, @@ -19,6 +19,16 @@ use kernel::{ pub(crate) struct CompletionFence; +unsafe extern "C" { + fn asahi_g17_signal_fence(fence: *mut bindings::dma_fence); +} + +fn signal_exported(fence: &dyn RawDmaFence) { + // SAFETY: The borrowed fence remains owned throughout synchronous signaling. + // Ready completion scopes are drained iteratively after its lock is released. + unsafe { asahi_g17_signal_fence(fence.raw()) }; +} + #[vtable] impl FenceOps for CompletionFence { fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { @@ -48,106 +58,274 @@ pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { Ok(Fence::from_fence(&fence)) } -/// All command members of an ioctl share its work-state node and aggregate fence. +/// The existing aggregate-fence allocation owns the existing submission state. +/// Submission never owns this fence, so exported references cannot form a cycle. +pub(crate) struct ExportFence { + source: Option>, + scoped: bool, +} +static EXPORT_OPS: AtomicUsize = AtomicUsize::new(0); + +#[vtable] +impl FenceOps for ExportFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { c_str!("asahi") } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("g17-aggregate") + } +} + +#[inline(never)] +fn export_fallback(contexts: &FenceContexts) -> Result> { + let fence = contexts.new_fence(0, ExportFence { source: None, scoped: false })?; + // SAFETY: Unique, initialized and still private. Only this constructor + // publishes ExportFence objects; its payload is immutable after sharing. + let ops = unsafe { + (*fence.raw()).context = bindings::dma_fence_context_alloc(1); + (*fence.raw()).ops as usize + }; + let previous = EXPORT_OPS.load(Ordering::Acquire); + if previous == 0 { + match EXPORT_OPS.compare_exchange(0, ops, Ordering::AcqRel, Ordering::Acquire) { + Ok(_) => {}, + Err(previous) if previous == ops => {}, + Err(_) => return Err(EIO), + } + } else if previous != ops { + return Err(EIO); + } + Ok(fence) +} + +/// Caller retains the leaf fence throughout lookup. Exact registered operations +/// identify the repr(C) FenceObject payload, not a driver or timeline name. +pub(super) unsafe fn exported_source_raw(raw: *mut bindings::dma_fence) + -> Option> +{ + let expected = EXPORT_OPS.load(Ordering::Acquire); + if expected == 0 { return None; } + // SAFETY: The caller owns a live fence reference. ExportFence has a unique + // timeline callback and nontrivial payload drop. No cast precedes the exact + // registered vtable check. FenceObject begins with its dma_fence. + let object = unsafe { + if (*raw).ops as usize != expected { return None; } + &*(raw as *const FenceObject) + }; + object.source.as_ref().filter(|source| source.firmware_source).cloned() +} + +struct State { + work: Option>, + ready: Option, +} + +// One Member per parsed command, plus the enqueue guard. Exported references +// retain the allocation but do not contribute to either of these counters. +kernel::static_assert!(crate::file::MAX_COMMANDS_PER_SUBMISSION < u8::MAX as u32); + /// The initial member keeps the aggregate unsignalled until enqueue finishes. #[pin_data] pub(crate) struct Submission { - fence: UserFence, - /// Successful retirement drops only this bookkeeping owner; runtime owners - /// and failed submissions keep their work-state allocation. #[pin] - work_state: Mutex>>, - remaining: AtomicU32, + state: Mutex, + remaining: AtomicU8, error: AtomicI32, + /// Live Member objects, plus the enqueue guard, independently of signalling. + owners: AtomicU8, + firmware_source: bool, + /// Set under the device mutex when the only member's successful retirement + /// is queued, before its fence is signalled off-lock: a consumer published + /// in between needs no wait on it. + retired: AtomicBool, } impl Submission { - fn new(fence: UserFence, context: Arc) -> Result> { - let work_state = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; - Arc::pin_init( - pin_init!(Self { - fence, - work_state <- new_mutex!(Some(work_state), "G17 submission work state"), - remaining: AtomicU32::new(1), - error: AtomicI32::new(0), - }), - GFP_KERNEL, - ) - } - - pub(crate) fn fence(&self) -> Fence { - Fence::from_fence(&self.fence) - } - - /// Add one command before enqueue ends. The parser bounds the count to 64. - pub(crate) fn member(self: &Arc) -> Member { - self.remaining.fetch_add(1, Ordering::Relaxed); - Member { - submission: self.clone(), - complete: AtomicBool::new(false), - } + fn new(context: Arc, firmware_source: bool) -> Result> { + let work = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + Self::with_work(Some(work), firmware_source) } - - fn member_done(&self, result: Result) { - if let Err(error) = result { - let _ = self.error.compare_exchange( - 0, - error.to_errno(), - Ordering::AcqRel, - Ordering::Acquire, - ); - } + fn with_work(work: Option>, firmware_source: bool) -> Result> { + Arc::pin_init(pin_init!(Self { + state <- new_mutex!(State { work, ready: None }, "G17 submission work state"), + remaining: AtomicU8::new(1), + error: AtomicI32::new(0), + owners: AtomicU8::new(1), + firmware_source, + retired: AtomicBool::new(false), + }), GFP_KERNEL) + } + pub(super) fn source_status(&self) -> Result { + let complete = self.remaining.load(Ordering::Acquire) == 0 + || self.retired.load(Ordering::Acquire); + let error = self.error.load(Ordering::Acquire); + if error != 0 { return Err(super::dependency::inherited_error(Error::from_errno(error))); } + Ok(complete) + } + pub(super) fn same_vm(&self, status: &Arc) -> bool { + self.state.lock().work.as_ref().is_some_and(|work| Arc::ptr_eq(work.status(), status)) + } + pub(super) fn ready(&self) -> Option { self.state.lock().ready.clone() } + fn set_ready(&self, ready: Fence) { + if !self.firmware_source { return; } + let mut state = self.state.lock(); + if state.work.is_some() && state.ready.is_none() { state.ready = Some(ready); } + } + fn note_error(&self, error: Error) -> bool { + self.error.compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire).is_ok() + } + fn release_state(&self) { + let old = { + let mut state = self.state.lock(); + (state.work.take(), state.ready.take()) + }; + drop(old); + } + fn owner_done(&self) { + // Every hardware owner retains its Completion/Member through exact + // retirement or stop. Exported fence FDs must not retain failed pins. + if self.owners.fetch_sub(1, Ordering::AcqRel) == 1 { self.release_state(); } + } + fn member_done(&self, fence: &UserFence, result: Result) { + if let Err(error) = result { self.note_error(error); } if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { let error = self.error.load(Ordering::Acquire); - if error != 0 { - self.fence.set_error(Error::from_errno(error)); - } else { - // Scheduler cleanup can wait behind submission on the same worker. - // Once every member and enqueue have succeeded, retained packets - // must not prevent new submissions from acquiring work-state nodes. - let retired = self.work_state.lock().take(); - drop(retired); - } - self.fence.signal(); + if error != 0 { fence.set_error(Error::from_errno(error)); } + else { self.release_state(); } + if fence.scoped { signal_exported(fence); } else { fence.signal(); } } } +} +#[derive(Clone)] +pub(crate) struct Aggregate(UserFence); +impl Aggregate { + fn source(&self) -> &Arc { + // Only Outputs::publish constructs Aggregate, after setting the payload. + self.0.source.as_ref().expect("published aggregate has submission state") + } + fn fence(&self) -> Fence { Fence::from_fence(&self.0) } + pub(crate) fn member(&self) -> Member { + self.source().remaining.fetch_add(1, Ordering::Relaxed); + self.source().owners.fetch_add(1, Ordering::Relaxed); + Member { aggregate: self.clone(), complete: AtomicBool::new(false) } + } fn finish_enqueue(&self, result: Result) { - // A dropped, never-enqueued member reports cancellation. The ioctl's actual - // failure takes precedence, while published members still retain their work. - if let Err(error) = result { - self.error.store(error.to_errno(), Ordering::Release); - } - self.member_done(result); + if let Err(error) = result { self.source().error.store(error.to_errno(), Ordering::Release); } + self.source().member_done(&self.0, result); + self.source().owner_done(); } } -/// One command's contribution to its submission, including construction failure. pub(crate) struct Member { - submission: Arc, + aggregate: Aggregate, complete: AtomicBool, } - impl Member { + pub(super) fn needs_source_ready(&self) -> bool { self.aggregate.source().firmware_source } pub(crate) fn work_state(&self) -> Result> { - self.submission - .work_state - .lock() - .as_ref() - .cloned() - .ok_or(ECANCELED) + self.aggregate.source().state.lock().work.clone().ok_or(ECANCELED) + } + pub(super) fn is_source(&self, source: &Arc) -> bool { + self.aggregate.source().firmware_source && Arc::ptr_eq(self.aggregate.source(), source) + } + pub(super) fn set_source_ready(&self, ready: Fence) { self.aggregate.source().set_ready(ready); } + pub(super) fn note_error(&self, error: Error) -> bool { self.aggregate.source().note_error(error) } + pub(super) fn note_retired(&self) { + let source = self.aggregate.source(); + if source.firmware_source { source.retired.store(true, Ordering::Release); } } - pub(crate) fn complete(&self, result: Result) { if !self.complete.swap(true, Ordering::AcqRel) { - self.submission.member_done(result); + self.aggregate.source().member_done(&self.aggregate.0, result); } } } +#[kunit_tests(asahi_g17_aggregate_promise)] +mod tests { + use super::*; + use kernel::sync::LockClassKey; + static KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + + fn contexts() -> FenceContexts { + FenceContexts::new(1, c_str!("asahi_promise_test"), KEY).unwrap() + } + fn aggregate(contexts: &FenceContexts) -> Aggregate { + let mut fence = export_fallback(contexts).unwrap(); + fence.source = Some(Submission::with_work(None, true).unwrap()); + Aggregate(fence.into()) + } + + #[test] + fn exact_export_identity_recognizes_unpublished_source() { + let contexts = contexts(); + let aggregate = aggregate(&contexts); + let plain = independent(&contexts).unwrap(); + // SAFETY: Both references remain owned throughout the checked lookup. + assert!(unsafe { exported_source_raw(plain.raw()) }.is_none()); + let source = unsafe { exported_source_raw(aggregate.0.raw()) }.unwrap(); + assert!(Arc::ptr_eq(&source, aggregate.source())); + assert_eq!(source.source_status().unwrap(), false); + aggregate.finish_enqueue(Ok(())); + assert_eq!(source.source_status().unwrap(), true); + } + + #[test] + fn failed_export_releases_state_after_last_runtime_owner() { + let contexts = contexts(); + let aggregate = aggregate(&contexts); + let source = aggregate.source().clone(); + source.state.lock().ready = Some(Fence::from_fence(&independent(&contexts).unwrap())); + let member = aggregate.member(); + aggregate.finish_enqueue(Ok(())); + member.complete(Err(EIO)); + assert_eq!(super::super::job::fence_status(&aggregate.fence()), EIO.to_errno()); + assert!(source.state.lock().ready.is_some()); + drop(member); + // Keep both the exported fence and source state alive, as userspace can. + assert!(source.state.lock().ready.is_none()); + assert_eq!(source.owners.load(Ordering::Acquire), 0); + } + + #[test] + fn actual_enqueue_error_wins_over_dropped_member_cancellation() { + let aggregate = aggregate(&contexts()); + drop(aggregate.member()); + aggregate.finish_enqueue(Err(ENOMEM)); + assert_eq!(super::super::job::fence_status(&aggregate.fence()), ENOMEM.to_errno()); + } + + #[test] + fn rejected_private_fence_never_exports_source_state() { + let fence = export_fallback(&contexts()).unwrap(); + fence.set_error(ENOMEM); + fence.signal(); + // SAFETY: The unique fence owns its initialized allocation. + assert!(unsafe { exported_source_raw(fence.raw()) }.is_none()); + } + + #[test] + fn maximum_submission_keeps_all_members_and_enqueue_guard() { + let aggregate = aggregate(&contexts()); + let source = aggregate.source().clone(); + let mut members: [Option; crate::file::MAX_COMMANDS_PER_SUBMISSION as usize] = + core::array::from_fn(|_| Some(aggregate.member())); + assert_eq!(source.remaining.load(Ordering::Acquire), 65); + assert_eq!(source.owners.load(Ordering::Acquire), 65); + aggregate.finish_enqueue(Ok(())); + for member in &members { + assert_eq!(super::super::job::fence_status(&aggregate.fence()), 0); + member.as_ref().unwrap().complete(Ok(())); + } + assert!(super::super::job::fence_status(&aggregate.fence()) > 0); + assert_eq!(source.remaining.load(Ordering::Acquire), 0); + for member in &mut members { drop(member.take()); } + assert_eq!(source.owners.load(Ordering::Acquire), 0); + } +} impl Drop for Member { fn drop(&mut self) { self.complete(Err(ECANCELED)); + self.aggregate.source().owner_done(); } } @@ -155,83 +333,73 @@ fn install(outputs: impl Iterator, fence: &Fence) { for mut sync in outputs { if let Some(chain) = sync.chain_fence.take() { sync.syncobj.add_point(chain, fence, sync.timeline_value); - } else { - sync.syncobj.replace_fence(Some(fence)); - } + } else { sync.syncobj.replace_fence(Some(fence)); } } } -/// Keeps resolved output syncobjs reachable on every fallible submission path. -/// Input syncobjs stay in the same vector, avoiding a separate output allocation. pub(crate) struct Outputs { - fallback: UserFence, + fallback: Option>, syncs: KVec, input_count: usize, - submission: Option>, + submission: Option, result: Result, + cumulative: Option, } - impl Outputs { - pub(crate) fn new( - contexts: &FenceContexts, - mut syncs: KVec, - input_count: usize, - allocation_failure: &Fence, - ) -> Result { - if input_count > syncs.len() { - return Err(EINVAL); - } - let fallback = match independent(contexts) { + pub(crate) fn new(contexts: &FenceContexts, mut syncs: KVec, + input_count: usize, allocation_failure: &Fence) -> Result + { + if input_count > syncs.len() { return Err(EINVAL); } + let mut cumulative = syncs.iter_mut().find_map(|sync| sync.cumulative.take()); + let fallback = match export_fallback(contexts) { Ok(fence) => fence, Err(error) => { - // Syncobjs and timeline-chain storage are already resolved. - // No work has been accepted; publish the pre-signaled ENOMEM - // fence without another allocation before returning the error. - install(syncs.drain(input_count..), allocation_failure); + if let Some(bundle) = &mut cumulative { + bundle.fail(allocation_failure); + } else { + install(syncs.drain(input_count..), allocation_failure); + } return Err(error); } }; - Ok(Self { - fallback, - syncs, - input_count, - submission: None, - result: Err(ECANCELED), - }) - } - - pub(crate) fn inputs(&self) -> &[SyncItem] { - &self.syncs[..self.input_count] + Ok(Self { fallback: Some(fallback), syncs, input_count, + submission: None, result: Err(ECANCELED), cumulative }) } - - /// Publish the actual aggregate before enqueueing any command. - pub(crate) fn publish(&mut self, context: Arc) -> Result> { - let submission = Submission::new(self.fallback.clone(), context)?; - install(self.syncs.drain(self.input_count..), &submission.fence()); + pub(crate) fn inputs(&self) -> &[SyncItem] { &self.syncs[..self.input_count] } + pub(crate) fn publish(&mut self, context: Arc, firmware_source: bool) -> Result { + let source = Submission::new(context, firmware_source)?; + let mut fence = self.fallback.take().ok_or(EINVAL)?; + fence.source = Some(source); + fence.scoped = self.cumulative.is_some(); + let submission = Aggregate(fence.into()); self.submission = Some(submission.clone()); + if let Some(bundle) = &mut self.cumulative { + bundle.publish(&submission.fence())?; + self.syncs.truncate(self.input_count); + } else { + install(self.syncs.drain(self.input_count..), &submission.fence()); + } Ok(submission) } - - /// The caller first records terminal VM errors and settles prior failed work. - pub(crate) fn finish(mut self, result: Result) -> Result { - self.result = result; - result - } + pub(crate) fn finish(mut self, result: Result) -> Result { self.result = result; result } } - impl Drop for Outputs { fn drop(&mut self) { if let Some(submission) = self.submission.take() { + if let Some(bundle) = &mut self.cumulative { + bundle.fail(&submission.fence()); + } submission.finish_enqueue(self.result); return; } - if let Err(error) = self.result { - self.fallback.set_error(error); + if let Some(fallback) = self.fallback.take() { + if let Err(error) = self.result { fallback.set_error(error); } + if let Some(bundle) = &mut self.cumulative { + bundle.fail(&Fence::from_fence(&fallback)); + } else { + install(self.syncs.drain(self.input_count..), &Fence::from_fence(&fallback)); + } + fallback.signal(); } - install( - self.syncs.drain(self.input_count..), - &Fence::from_fence(&self.fallback), - ); - self.fallback.signal(); } } diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 3b31c3cead064..9015dc82e4076 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -670,10 +670,18 @@ pub(crate) struct Owner { enum LeasePhase { Preparing, Ready, + /// No reference since the given time. The registration stays live so that + /// the owner's next render reuses it without a control-ring release; the + /// release becomes pending when the linger ends. + Lingering(u64), ReleaseSent(u32), Released, } +/// How long an idle registration of an open owner may stay live: the +/// firmware's effort idle hold. +pub(crate) const LINGER_NS: u64 = 16_000_000; + /// References to one prepopulated owner: preparation and two engine commands /// per render. Published releases do not gate reacquisition; their consumption /// witnesses are carried into the successor. @@ -707,10 +715,14 @@ impl Lease { } let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; let same_owner = previous.owner == owner; + // A lingering registration was never released: it continues like an + // overlap of the owner's previous render. let overlapping = same_owner - && previous.phase == LeasePhase::Ready - && previous.references >= 2 - && previous.commands >= 2; + && match previous.phase { + LeasePhase::Ready => previous.references >= 2 && previous.commands >= 2, + LeasePhase::Lingering(_) => true, + _ => false, + }; let reacquire = previous.reacquirable(); if !overlapping && !reacquire { return Err(EBUSY); @@ -772,16 +784,35 @@ impl Lease { Err(EIO) } /// Both engines have retired this render's exact retained submission. + /// With `linger`, a registration left without references stays live + /// instead of making its final release pending. pub(crate) fn complete_render( &mut self, + linger: bool, ) -> Result> { if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { return Err(EINVAL); } self.references -= 2; self.commands -= 2; + if self.references == 0 && linger { + self.phase = LeasePhase::Lingering(super::now_ns().max(1)); + } Ok(self.pending_release()) } + /// The time since which an idle registration lingers, if it does. + pub(crate) fn lingering(&self) -> Option { + match self.phase { + LeasePhase::Lingering(since) => Some(since), + _ => None, + } + } + /// Ends a linger; the final release becomes pending. + pub(crate) fn end_linger(&mut self) { + if matches!(self.phase, LeasePhase::Lingering(_)) { + self.phase = LeasePhase::Ready; + } + } pub(crate) fn pending_release(&self) -> Option { (self.phase == LeasePhase::Ready && self.references == 0).then(|| { super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0cb7304e30677..b1afdad816b6b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -12,7 +12,7 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; use super::queue::CommandTag; use super::{ - clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, }; @@ -507,8 +507,9 @@ impl ComputeDescriptor { const UNK_FBA: u32 = 0xe0a0_0001; const UNK_FC5: u8 = 0x9f; - /// Writes the complete descriptor. - pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { + /// Fills a descriptor cleared by `Object::initialize`. The unused firmware + /// fields and padding retain that zero initialization. + pub(crate) fn write_zeroed(&mut self, args: &ComputeArgs) -> Result { if args.cdm_end_va < args.cdm_va || (args.sampler_heap_va == 0) != (args.sampler_count == 0) || args.sampler_heap_va & 7 != 0 @@ -522,7 +523,6 @@ impl ComputeDescriptor { let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; let sampler_max = sampler_max(args.sampler_count)?; - clear(self); self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 4f3a70671eb87..ca2b6604a302b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -278,6 +278,20 @@ impl RegisterArray { self.size = (size_of_val(writes) as u32) << 16 | count as u32; Ok(()) } + + /// Stores `writes` without the entry at `skip`, exactly as [`Self::set`] stores the + /// remaining writes, without first copying them elsewhere. + pub(crate) fn set_without(&mut self, self_va: u64, writes: &[RegisterWrite], skip: usize) -> Result { + let count = writes.len().checked_sub(1).ok_or(EINVAL)?; + if skip > count || count > REGISTER_ARRAY_ENTRIES { + return Err(EINVAL); + } + self.entries[..skip].copy_from_slice(&writes[..skip]); + self.entries[skip..count].copy_from_slice(&writes[skip + 1..]); + self.self_va = self_va; + self.size = ((count * size_of::()) as u32) << 16 | count as u32; + Ok(()) + } } /// Binding of one [`RegisterArray`] to a kick entry. diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 0e270d40392a8..aa8693bf7140b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -1110,9 +1110,7 @@ impl TaDescriptor { } else if scs_register { self.registers.set(array_va, ®isters)?; } else { - let mut trimmed = [registers[0]; TA_REGISTERS]; - trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); - self.registers.set(array_va, &trimmed)?; + self.registers.set_without(array_va, ®isters, TA_SCS_ENABLE_INDEX)?; } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; @@ -1456,10 +1454,7 @@ impl FragmentSlot { } else if scs_register { desc.arrays[0].set(arrays[0], ®isters)?; } else { - let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; - trimmed[FRAGMENT_SCS_ENABLE_INDEX..] - .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); - desc.arrays[0].set(arrays[0], &trimmed)?; + desc.arrays[0].set_without(arrays[0], ®isters, FRAGMENT_SCS_ENABLE_INDEX)?; } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs index 20daaab85b832..4d9e99d9a9cc0 100644 --- a/drivers/gpu/drm/asahi/g17/hazard.rs +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -3,11 +3,21 @@ //! Attachment overlap ordering across physical render pairs and shared images. //! Tile stores can overlap even when the application's pixel regions do not. //! Keep independent attachments concurrent; order only conflicting live spans. +//! +//! Within one VM, a render of another logical owner is ordered at publication: +//! the device mutex totally orders publications, and the later render's +//! fragment kick waits in firmware for the newest overlapping fragment kick of +//! every other pair. Admission never waits for such an overlap. Only a +//! cross-VM overlap through a shared object waits on the host for retirement, +//! so that one VM's fault can never hold another VM's kick in firmware. use super::job::Order; const MAX_SPANS: usize = 24; pub(crate) const WAIT_MS: u32 = 1000; +/// Firmware barriers one render publication may add for same-VM overlaps, +/// one per other pair. +pub(crate) const PUBLICATION_BARRIERS: usize = 7; #[derive(Clone, Copy)] struct Span { @@ -56,7 +66,12 @@ impl Footprint { self.shared |= self.count != before; } - fn overlaps(&self, other: &Self) -> bool { + /// Whether a publication orders an overlap with `other` in firmware. + pub(crate) fn same_vm(&self, other: &Self) -> bool { + self.vm == other.vm + } + + pub(crate) fn overlaps(&self, other: &Self) -> bool { if self.vm != other.vm && !(self.shared && other.shared) { return false; } @@ -86,6 +101,13 @@ pub(crate) struct Live<'a> { /// A queue prefix already orders covered predecessors. Pending arrivals order /// different owners fairly without imposing order on unrelated attachments. +/// +/// A same-VM overlap with another owner's admitted render needs no wait; its +/// publication carries one barrier per other pair. That count stays within +/// [`PUBLICATION_BARRIERS`] because such a render is admitted only while fewer +/// than `PUBLICATION_BARRIERS + 1` other pairs of its VM hold admitted renders: +/// of any set of renders overlapping one publication, the last admitted saw all +/// the others live. pub(crate) fn plan<'a>( owner: u64, prefix: Option, @@ -94,10 +116,29 @@ pub(crate) fn plan<'a>( live: impl IntoIterator>, ) -> Plan { let mut selected = None; + let mut ordered = false; + let mut pairs = [0u8; PUBLICATION_BARRIERS]; + let mut count = 0; + let mut crowded = false; for entry in live { + let foreign = entry.owner != owner && footprint.same_vm(entry.footprint); + if let Some(slot) = entry.slot.filter(|_| foreign) { + if !pairs[..count].contains(&slot) { + if count == PUBLICATION_BARRIERS { + crowded = true; + } else { + pairs[count] = slot; + count += 1; + } + } + } if entry.quarantined || !footprint.overlaps(entry.footprint) { continue; } + if foreign { + ordered |= entry.slot.is_some(); + continue; + } let Some(slot) = entry.slot else { if entry.owner != owner && ticket.is_none_or(|mine| entry.ticket < mine) { return Plan::Wait; @@ -116,5 +157,8 @@ pub(crate) fn plan<'a>( Some(_) => return Plan::Wait, } } + if ordered && crowded { + return Plan::Wait; + } selected.map_or(Plan::Any, Plan::Pair) } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 10af261aeb3ac..7e0c36ece43ad 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -38,6 +38,26 @@ pub(crate) struct ComputeCompletion<'a> { offset: usize, } +/// Records in the compute completion ring that the host scans. +pub(super) const COMPUTE_COMPLETION_SLOTS: usize = { + let slots = abi::COMPLETION_RING_SIZE / 0x40; + if slots < 0x800 { slots } else { 0x800 } +}; + +/// Timestamp bits of a completion record's start and end words. +const COMPLETION_TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; + +/// What one completion-ring slot gave a scan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum CompletionFold { + /// No new observation of a pending request; the scan continues. + Passed, + /// Every request now has an observation. + Done, + /// A pending request's record whose payload the firmware is still writing. + Incomplete, +} + impl ComputeCompletion<'_> { pub(crate) fn read(&self, word: usize) -> Result { if word >= 8 { @@ -129,14 +149,6 @@ impl WorkPublication<'_> { ); } } - - fn commit(self) -> u32 { - self.copy_record(); - fence(Ordering::SeqCst); - self.producer.store(self.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - self.next - } } /// Session-long storage and mappings. Aliases are released before their backing objects. @@ -394,6 +406,12 @@ impl InitData { Ok(()) } + pub(super) fn compute_vm_cached(&self, vm: &mmu::Vm) -> bool { + self.compute_shared + .as_ref() + .is_some_and(|shared| vm.compute_pool_mappings_match(shared.id)) + } + pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { let shared = self.compute_shared.as_ref().ok_or(EIO)?; if vm.compute_pool_mappings_match(shared.id) { @@ -510,7 +528,7 @@ impl InitData { let engine = match record.data_master { 0 => DataMaster::Tiling, 1 => DataMaster::Fragment, - 2 => DataMaster::Compute, + // Compute publications reserve their slots through the port. _ => return Err(EINVAL), }; let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; @@ -563,16 +581,6 @@ impl InitData { Ok(values) } - /// Inner queue records must be visible before this outer publication. The returned - /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work( - &self, - priority: u8, - record: &super::fw::channels::WorkSlot, - ) -> Result { - Ok(self.work_publication(priority, record)?.commit()) - } - /// Validate both destinations before exposing fragment, then tiling. The accounting token /// is committed after copying fragment; late tiling state is published before its slot. /// Deferred pairs retain both slot bodies without exposing either producer. @@ -671,21 +679,118 @@ impl InitData { Ok(()) } + /// Raw views of the memory publishers share without the device mutex: every work ring, + /// the completion stamps, the QoS table and the recovery state word. They stay valid for + /// this session. + pub(super) fn port_views( + &self, + mailbox: *const super::Mailbox, + ) -> Result { + use super::fw::{channels, queue::{DataMaster, Policy}}; + let ring_of = |priority: u8, engine: DataMaster| -> Result { + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; + let base = self.state_offset(Role::Primary) + abi::work_state_offset(channel); + let state = self.object(self.cluster)?; + let word = |offset: usize| -> Result<*const AtomicU32> { + Ok(state.word(base + offset)?) + }; + Ok(super::port::RingView { + slots: self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel), + channels::WORK_RING_SLOTS as usize + * core::mem::size_of::(), + )?, + consumers: [ + word(abi::ChannelState::CONSUMER)?, + word(abi::ChannelState::UNK_10)?, + ], + producer: word(abi::ChannelState::PRODUCER)?, + }) + }; + let rings = |engine: DataMaster| -> Result<[super::port::RingView; Policy::PRIORITIES as usize]> { + let mut views = [ring_of(0, engine)?; Policy::PRIORITIES as usize]; + for (priority, view) in views.iter_mut().enumerate().skip(1) { + *view = ring_of(priority as u8, engine)?; + } + Ok(views) + }; + Ok(super::port::Views { + compute: rings(DataMaster::Compute)?, + tiling: rings(DataMaster::Tiling)?, + fragment: rings(DataMaster::Fragment)?, + stamps: self.object(self.stamps[0])?.word(0)?, + qos: self + .qos()? + .pointer(0, core::mem::size_of::())?, + recovery: self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?, + mailbox, + }) + } + /// Snapshot the shared compute completion ring once per service transaction. The queue /// scanner latches exact identities; firmware retains sole ownership of these slots. - pub(super) fn scan_compute_completions( + /// Folds the compute completion records the firmware appended since `start`, in ring + /// order. `seen` holds every slot's control word as of the previous pass over it: the + /// first slot still holding it is where the firmware writes next, and the pass ends there + /// instead of reading the rest of the ring. A pass also ends once `record` has found every + /// request, or at a requested record whose payload is still being written, which the next + /// pass reads again. Returns the slot the next pass starts at. + pub(super) fn fold_compute_completions( + &self, + start: usize, + seen: &mut [u64], + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, + ) -> Result { + let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } + fence(Ordering::Acquire); + let completion = |slot: usize| ComputeCompletion { + object, + offset: slot * 0x40, + }; + super::queue::compute::retirement::walk_completions( + start, + seen, + |slot| completion(slot).read(0), + |slot| record(completion(slot)), + ) + } + + /// Folds every slot of the compute completion ring and records its control word in + /// `seen`. For a pass that stopped before a record the firmware must already have + /// written. Returns the slot after the newest record (latest end timestamp), where the + /// firmware writes next. + pub(super) fn resync_compute_completions( &self, + seen: &mut [u64], mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { + ) -> Result { let object = self.object(self.completion_rings[0])?; + if seen.len() != COMPUTE_COMPLETION_SLOTS { + return Err(EINVAL); + } fence(Ordering::Acquire); - for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - record(ComputeCompletion { + let mut newest = None; + for slot in 0..COMPUTE_COMPLETION_SLOTS { + let completion = ComputeCompletion { object, offset: slot * 0x40, - })?; + }; + let control = completion.read(0)?; + seen[slot] = control; + if control != 0 { + let end = completion.read(5)? & COMPLETION_TIME_MASK; + if newest.is_none_or(|(_, latest)| end > latest) { + newest = Some((slot, end)); + } + } + record(completion)?; } - Ok(()) + Ok(newest.map_or(0, |(slot, _)| (slot + 1) % COMPUTE_COMPLETION_SLOTS)) } fn object(&self, index: usize) -> Result<&KernelObject> { @@ -1517,17 +1622,22 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entries(&self) -> Result { + fn entry(&self, index: usize) -> Result<(u32, u32)> { + if index >= super::recovery::RECOVERY_RECORDS { + return Err(EINVAL); + } let cluster = self.object(self.cluster)?; - let base = self.status_b + abi::PrimaryStatusB::RECOVERY_INFO; + let offset = self.status_b + + abi::PrimaryStatusB::RECOVERY_INFO + + index * core::mem::size_of::(); fence(Ordering::Acquire); - let mut entries = super::recovery::Entries::default(); - for index in 0..256 { - let offset = base + index * core::mem::size_of::(); - let flags = cluster.word(offset)?.load(Ordering::Relaxed); - entries.include(flags); - } - Ok(entries) + let flags = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, flags))? + .load(Ordering::Relaxed); + let stamp = cluster + .word(offset + core::mem::offset_of!(abi::RecoveryInfoEntry, stamp))? + .load(Ordering::Relaxed); + Ok((flags, stamp)) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 3b47d361c2ef9..8175c992b8a62 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -14,7 +14,10 @@ use kernel::{ }; pub(super) const TIMEOUT_MS: kernel::time::Msecs = super::timeout::POLL_MS; -const RENDER_CREDITS: u32 = 32; +/// One logical render queue owns one physical pair, whose active ring holds +/// `RENDER_DEPTH` commands. A deeper scheduler would only park run_job in +/// admission until a retirement; the scheduler waits for the credit instead. +const RENDER_CREDITS: u32 = crate::hw::t8140::queues::RENDER_DEPTH as u32; const COMPUTE_CREDITS: u32 = 16; #[derive(Clone, Copy, PartialEq, Eq)] @@ -150,6 +153,7 @@ impl Packet { super::dependency::replay_status(fence_status(fence))?; } for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -183,6 +187,7 @@ impl Packet { for fence in &self.firmware { check_status(fence)?; } + self.completion.check_source_inputs()?; Ok(()) } @@ -262,6 +267,9 @@ fn check_status(fence: &Fence) -> Result { /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { /// Device detector retained by render completions after queue teardown. + fn input_sources(&self, _inputs: &[crate::file::SyncItem]) + -> Option> { None } + fn feed(&self) -> Option> { None } @@ -277,7 +285,9 @@ pub(crate) trait Backend: Send + Sync { fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. fn release_owner(&self); - fn prepare(&self, packet: &Packet) -> Option; + /// A fence to wait for before `publish`, signalled by the release of the capacity the + /// packet lacks. Called again once it signals. + fn prepare(&self, packet: &Arc) -> Option; fn publish(&self, packet: Arc) -> Result; fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); @@ -297,6 +307,7 @@ struct Timeout { backend: Arc, victim_renewals: u8, execution_renewals: u8, + shared_renewals: u8, packet: Arc, } @@ -321,6 +332,7 @@ impl sched::JobImpl for Job { backend: self.backend.clone(), victim_renewals: 0, execution_renewals: 0, + shared_renewals: 0, packet: self.packet.clone(), } } @@ -334,6 +346,13 @@ impl sched::JobImpl for Job { ) { return true; } + // Other work retiring means the GPU is shared, not hung. + if super::timeout::renew_shared( + job.packet.is_published(), super::recovery::since_retired(super::now_ns()), + &mut job.shared_renewals, + ) { + return true; + } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); if deferred { job.victim_renewals = job.victim_renewals.saturating_add(1); @@ -375,6 +394,18 @@ pub(crate) struct Fences { pub(crate) publication: Option>, } +impl Fences { + /// Scheduler::enqueue always supplies our explicit completion fence here. + /// It has no polling callback, so pending frontier checks need no lock. + pub(crate) fn status(&self) -> i32 { + unsafe extern "C" { + fn asahi_g17_completion_fence_status(fence: *mut bindings::dma_fence) -> i32; + } + // SAFETY: completed owns the live private fence throughout this read. + unsafe { asahi_g17_completion_fence_status(self.completed.raw()) } + } +} + impl Scheduler { pub(crate) fn new(dev: &AsahiDrmDevice, backend: Arc, engine: Engine) -> Result { let (credits, name) = match engine { @@ -429,7 +460,10 @@ impl Scheduler { }; // All fallible scheduler setup precedes early hardware publication. The // scheduler still owns watchdog, credits and cancellation after push. - if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + if !(early && self.backend.try_early(&packet)) { + if packet.completion.needs_source_ready() { + packet.completion.set_source_ready(ready.clone()); + } self.backend.count_backlog(&packet); } job.push(); diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index c337a8b298f19..a1f8f70f597d4 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -24,22 +24,38 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -/// Minimum render-channel capacity protected from compute-only allocation. -// Protect one tiling/fragment pair for each logical execution context, plus -// the two unpublished bootstrap reservations. The remaining ten compute -// channels are shared after exact retirement, rather than pinned per client. -const COMPUTE_RENDER_RESERVE: usize = 2 * crate::mmu::MAX_EXECUTION_CONTEXTS as usize + 2; -pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT - COMPUTE_RENDER_RESERVE; -/// Logical queues that can each hold a render pair: what the render reserve leaves beside the -/// two bootstrap reservations once the QIDs below `floor`, which the firmware keeps, are gone -/// (every execution context with no floor). +/// Render identities that must stay reachable for `queues` logical queues: +/// one tiling/fragment pair each, plus the two bootstrap reservations. +/// +/// Installed identities never change engine before processor stop, so a +/// compute channel permanently narrows what render can still install. Compute +/// may take a free QID only while every open logical queue, and +/// [`RENDER_HEADROOM`] queues opened later, can still install a dedicated +/// pair; logical-queue admission refuses a queue whose pair no longer fits. +pub(super) const fn render_reserve(queues: usize) -> usize { + let queues = queues + RENDER_HEADROOM; + let max = crate::mmu::MAX_EXECUTION_CONTEXTS as usize; + 2 * if queues < max { queues } else { max } + 2 +} +/// Logical queues that may still open after compute growth without losing a +/// dedicated render pair. +const RENDER_HEADROOM: usize = 24; +/// The render reserve with every execution context open. +const FULL_RENDER_RESERVE: usize = render_reserve(crate::mmu::MAX_EXECUTION_CONTEXTS as usize); +/// Compute channels that stay reachable with every execution context open. +#[cfg(CONFIG_KUNIT)] +pub(super) const COMPUTE_FLOOR: usize = QID_COUNT - FULL_RENDER_RESERVE; +/// Upper bound on distinct physical compute channels named by one kick. +pub(super) const COMPUTE_CHANNELS: usize = QID_COUNT; +/// Maximum logical queues after firmware-owned QIDs are excluded. pub(super) const fn queue_limit(floor: usize) -> u32 { - ((COMPUTE_RENDER_RESERVE - 2 - floor) / 2) as u32 + ((FULL_RENDER_RESERVE - 2 - floor) / 2) as u32 } const _: () = { assert!(queue_limit(0) == crate::mmu::MAX_EXECUTION_CONTEXTS); - assert!(COMPUTE_CHANNELS >= 4); + // Compute keeps this many channels even with every execution context open. + assert!(QID_COUNT - FULL_RENDER_RESERVE >= 4); assert!(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1); }; @@ -71,7 +87,6 @@ struct Identity { pub(crate) struct QueueIds { entries: [Option; QID_COUNT], initial_render: [Option; 2], - publication: u64, leased: usize, /// First QID the host may use ([`crate::g17::fw::Variant::qid_floor`]). floor: usize, @@ -82,10 +97,9 @@ impl QueueIds { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), initial_render: [None; 2], - publication: 0, leased: 0, // The reserve must keep render capacity above the floor. - floor: if floor < COMPUTE_RENDER_RESERVE { floor } else { return Err(EINVAL) }, + floor: if floor < FULL_RENDER_RESERVE { floor } else { return Err(EINVAL) }, }) } @@ -113,6 +127,53 @@ impl QueueIds { owner: u64, engine: DataMaster, preferred: Option, + ) -> Result { + self.reserve_for(owner, engine, preferred, crate::mmu::MAX_EXECUTION_CONTEXTS as usize) + } + + fn render_ids(&self) -> usize { + self.entries + .iter() + .flatten() + .filter(|entry| matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment)) + .count() + } + + /// Whether `queues` open logical queues can each still install a dedicated + /// render pair, beside the bootstrap reservations and installed compute. + pub(crate) fn render_fits(&self, queues: usize) -> bool { + self.render_ids() + (QID_COUNT - self.floor - self.leased) >= 2 * queues + 2 + } + + /// Whether a fresh compute identity fits beside the render reserve of + /// `queues` open logical queues, without reserving it. + pub(crate) fn compute_fits(&self, queues: usize) -> bool { + let initial = self.leased == QID_COUNT - self.floor + && self.initial_render.iter().flatten().any(|id| self.can_cancel_unpublished(*id)); + self.compute_room(queues, usize::from(initial)) + } + + fn compute_room(&self, queues: usize, replacement: usize) -> bool { + // Existing render identities already satisfy this reservation. Counting + // them twice can refuse compute even while physical QIDs remain free. + // Replacing an initial reservation consumes one render identity. + // Include that transition when protecting render's minimum, and + // count the replacement as available compute capacity. + let render = self.render_ids(); + QID_COUNT - self.floor - self.leased + replacement + > render_reserve(queues) + .min(FULL_RENDER_RESERVE - self.floor) + .saturating_sub(render - replacement) + } + + /// As [`Self::reserve`], with the compute reserve sized for `queues` open + /// logical queues instead of every possible execution context. + pub(crate) fn reserve_for( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + queues: usize, ) -> Result { if owner == 0 { return Err(EINVAL); @@ -129,10 +190,8 @@ impl QueueIds { Err(EINVAL) }; } - // The QIDs below the floor are never the host's; the reserve shrinks - // with them, so the compute channel count stays the same. + // The QIDs below the floor are never available to the host. let total = QID_COUNT - self.floor; - let reserve = COMPUTE_RENDER_RESERVE - self.floor; let initial = if engine == DataMaster::Compute && self.leased == total { self.initial_render .iter() @@ -145,27 +204,9 @@ impl QueueIds { } else { None }; - // Existing render identities already satisfy this reservation. Counting - // them twice can refuse compute even while physical QIDs remain free. // Published identities always retain their original lifetime rules. - if engine == DataMaster::Compute { - let render = self - .entries - .iter() - .flatten() - .filter(|entry| { - matches!(entry.id.engine, DataMaster::Tiling | DataMaster::Fragment) - }) - .count(); - // Replacing an initial reservation consumes one render identity. - // Include that transition when protecting render's minimum, and - // count the replacement as available compute capacity. - let replacement = usize::from(initial.is_some()); - if total - self.leased + replacement - <= reserve.saturating_sub(render - replacement) - { - return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); - } + if engine == DataMaster::Compute && !self.compute_room(queues, usize::from(initial.is_some())) { + return Err(crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeReserve)); } let index = preferred .map(|index| usize::from(index) + self.floor) @@ -196,13 +237,6 @@ impl QueueIds { Ok(id) } - /// Ordered under the device mutex; no atomic operation on the submission path. Compute and - /// render publications share it, so a watchdog can order work across both engines. - pub(crate) fn next_publication(&mut self) -> Result { - self.publication = self.publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.publication) - } - /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { @@ -249,10 +283,10 @@ mod accounting_tests { #[test] fn render_identities_satisfy_the_compute_reserve() { let mut ids = table(); - for owner in 1..=COMPUTE_RENDER_RESERVE as u64 { + for owner in 1..=FULL_RENDER_RESERVE as u64 { ids.reserve(owner, DataMaster::Tiling, None).unwrap(); } - for owner in COMPUTE_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { + for owner in FULL_RENDER_RESERVE as u64 + 1..=QID_COUNT as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.leased, QID_COUNT); @@ -263,14 +297,49 @@ mod accounting_tests { #[test] fn compute_cannot_consume_the_render_minimum() { let mut ids = table(); - for owner in 1..=(QID_COUNT - COMPUTE_RENDER_RESERVE) as u64 { + for owner in 1..=(QID_COUNT - FULL_RENDER_RESERVE) as u64 { ids.reserve(owner, DataMaster::Compute, None).unwrap(); } assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); let tiling = ids.reserve(1001, DataMaster::Tiling, None).unwrap(); assert_eq!(ids.reserve(1000, DataMaster::Compute, None), Err(ENOSPC)); ids.cancel_unpublished(tiling).unwrap(); - assert_eq!(ids.leased, QID_COUNT - COMPUTE_RENDER_RESERVE); + assert_eq!(ids.leased, QID_COUNT - FULL_RENDER_RESERVE); + } + + #[test] + fn dynamic_compute_growth_preserves_render_headroom_above_firmware_floor() { + for floor in [0, 4] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = 2; + let mut owner = 1; + while ids.compute_fits(queues) { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + owner += 1; + } + assert_eq!(ids.leased, QID_COUNT - floor - render_reserve(queues)); + assert_eq!(ids.reserve_for(owner, DataMaster::Compute, None, queues), Err(ENOSPC)); + assert!(ids.render_fits(queues + RENDER_HEADROOM)); + assert!(!ids.render_fits(queues + RENDER_HEADROOM + 1)); + } + } + + #[test] + fn full_logical_queue_budget_keeps_compute_floor_and_excludes_firmware_ids() { + for floor in [0, 4] { + let mut ids = KBox::init(QueueIds::new(floor), GFP_KERNEL).unwrap(); + let queues = queue_limit(floor) as usize; + for owner in 1..=COMPUTE_FLOOR as u64 { + let id = ids.reserve_for(owner, DataMaster::Compute, None, queues).unwrap(); + assert!(usize::from(id.qid()) >= floor); + ids.publish(id).unwrap(); + } + assert!(!ids.compute_fits(queues)); + assert!(ids.render_fits(queues)); + assert!(!ids.render_fits(queues + 1)); + } } #[test] diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ae78efaad1a95..c63d5a3258314 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -12,11 +12,16 @@ //! //! # Locking //! -//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all -//! firmware-shared state, including the bounded waits for the firmware that publication needs. -//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state -//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. - +//! `Shared::state` is the device lock. It serializes binding, recovery, the event worker and all +//! firmware-shared state that has no narrower owner. Steady-state compute publication into a +//! bound queue takes only that queue's cell lock and the port's locks ([`port`]), reached under +//! a preparation lease. Completions are signalled after dropping the device lock. Lock order: +//! `Shared::state` -> compute queue cell (one at a time) -> port locks -> availability domains; +//! `Shared::state` -> UAT shared state -> VM exec lock -> GEM reservation. `RoleState::ack` +//! nests inside nothing. Capacity waits block on an availability domain signalled by the +//! release that provides the capacity, never on a sleep. + +mod availability; mod buffer; mod channel; mod command; @@ -25,6 +30,9 @@ mod completion; pub(crate) mod context; mod data_segment; mod dependency; +mod sync_input; +pub(crate) mod sync_output; +mod trace; #[cfg(CONFIG_DRM_ASAHI_G15)] pub(crate) use dependency::inherited_error; mod event; @@ -39,6 +47,7 @@ mod job; mod capacity; mod kick; mod object; +mod port; pub(crate) mod power; mod preparation; mod qos; @@ -55,6 +64,7 @@ use core::any::Any; use core::ops::Range; use core::sync::atomic::{ AtomicBool, + AtomicU32, Ordering, // }; @@ -394,13 +404,48 @@ impl rtkit::Operations for RoleOps { } } +/// The RTKit of one coprocessor. The mailbox driver serializes its own register writes; this +/// lock only serializes the exclusive borrows the RTKit API takes, so that publishers holding +/// different locks can ring the same doorbell. +#[pin_data] +struct Mailbox { + #[pin] + lock: Mutex<()>, + rtkit: core::cell::UnsafeCell>, +} + +// SAFETY: Every access to the RTKit goes through `with`, which holds the lock. +unsafe impl Sync for Mailbox {} + +impl Mailbox { + fn new(rtkit: rtkit::RtKit) -> Result>> { + KBox::pin_init( + pin_init!(Self { + lock <- new_mutex!((), "g17::Mailbox::lock"), + rtkit: core::cell::UnsafeCell::new(rtkit), + }), + GFP_KERNEL, + ) + } + + fn with(&self, f: impl FnOnce(Pin<&mut rtkit::RtKit>) -> R) -> R { + let _guard = self.lock.lock(); + // SAFETY: The lock excludes every other borrow of the RTKit. + f(Pin::new(unsafe { &mut *self.rtkit.get() })) + } + + fn send(&self, endpoint: u8, message: u64) -> Result { + self.with(|rtkit| rtkit.send_message(endpoint, message)) + } +} + /// A running firmware coprocessor. /// /// Dropping it stops the CPU before RTKit and the crash log buffer go away. struct Coprocessor { cpu: regs::CpuControl, running: bool, - rtkit: rtkit::RtKit, + rtkit: Pin>, state: Arc, _crash_buffer: Option, } @@ -455,7 +500,7 @@ impl Coprocessor { Ok(Coprocessor { cpu: regs::CpuControl::new(pdev, index)?, running: false, - rtkit: Self::receiver(dev, index, &state)?, + rtkit: Mailbox::new(Self::receiver(dev, index, &state)?)?, state, _crash_buffer: crash_buffer, }) @@ -484,19 +529,20 @@ impl Coprocessor { fn boot(&mut self) -> Result { self.cpu.start()?; self.running = true; - Pin::new(&mut self.rtkit).boot() + self.rtkit.with(|rtkit| rtkit.boot()) } /// Starts the firmware endpoints and hands the firmware its initialization root. An /// acknowledgment message is awaited here; one in status A is the caller's to poll. fn init(&mut self, root_va: u64, ack: fw::variant::InitAck) -> Result { - let mut rtkit = Pin::new(&mut self.rtkit); - rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; - rtkit.as_mut().start_endpoint(EP_DOORBELL)?; - rtkit.send_message( - EP_FIRMWARE, - MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), - )?; + self.rtkit.with(|mut rtkit| { + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + ) + })?; match ack { fw::variant::InitAck::Message => self.state.wait_for_ack(), fw::variant::InitAck::Status => Ok(()), @@ -504,8 +550,9 @@ impl Coprocessor { } /// Sends a host notification to the firmware. - fn notify(&mut self, msg: u64) -> Result { - Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) + fn notify(&self, msg: u64) -> Result { + trace::message(trace::MAILBOX, msg); + self.rtkit.send(EP_DOORBELL, msg) } /// Stops the CPU. The firmware keeps its state, but no longer runs. @@ -572,6 +619,10 @@ struct Firmware { secondary: Option, primary: Coprocessor, bridge: Bridge, + /// Publication state shared with publishers that do not hold the device mutex. + port: Arc, + /// Fast-path submissions already accounted by the idle check. + seen_submissions: u64, rings: channel::Rings, recovery: recovery::State, effort: power::Effort, @@ -646,34 +697,50 @@ impl Firmware { } fn apply_effort(&mut self, effort: u32) -> Result { - Self::apply_shared_effort(&self.init, &mut self.effort, effort) + Self::apply_shared_effort(&self.init, &mut self.effort, &self.port, effort) } fn apply_shared_effort( init: &initdata::InitData, state: &mut power::Effort, + port: &port::Port, effort: u32, ) -> Result { if state.applied() == effort && init.effort()? == effort { return Ok(()); } init.publish_effort(effort)?; + trace::message(trace::EFFORT, u64::from(effort)); state.did_apply(effort); + port.set_effort_full(effort == power::FULL_EFFORT); Ok(()) } /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, - /// unretired work; quarantined work retaining mappings is excluded. + /// unretired work; quarantined work retaining mappings is excluded. Submissions published + /// without the device mutex count as activity, and a clear that races one is abandoned. fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - let effort = match self.effort.idle(now_ns, in_flight) { + let submissions = self.port.submissions(); + let submitted = submissions != self.seen_submissions; + self.seen_submissions = submissions; + if submitted { + self.effort.submit(); + } + let effort = match self.effort.idle(now_ns, in_flight || submitted) { power::IdleAction::None => None, power::IdleAction::Clear => { + self.port.set_effort_full(false); + if self.port.submissions() != submissions { + self.port.set_effort_full(true); + self.effort.submit(); + return Ok(Some(power::IDLE_HOLD_NS)); + } self.apply_effort(0)?; None } power::IdleAction::Recheck(delay) => Some(delay), }; - Ok(match (effort, self.idle_hold(now_ns, in_flight)) { + Ok(match (effort, self.idle_hold(now_ns, in_flight || submitted)) { (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), }) @@ -773,6 +840,21 @@ struct Shared { #[pin] feed_work: Work, preparations: Arc, + /// Compute admission waiting for a physical channel: signalled when a channel becomes + /// reusable or shareable, a QID or slot is returned, or a pool vacancy is witnessed. + channels: Arc, + /// Compute admission waiting for retirement-bound capacity (work states, ring and + /// in-flight room, drain witnesses, pool releases) or for publication to reopen. + progress: Arc, + /// Logical-queue creation waiting for a returned queue slot or execution context. + queues: Arc, + /// Render publication waiting for render capacity, a pair on loan, a work + /// state or firmware room: signalled when a pair, lease or work state + /// returns, an owner closes, a teardown parks, and on every event pass + /// while a render publisher waits (firmware consumption has no event). + renders: Arc, + /// Unbound compute publishers which can reuse a retired channel. + compute_demand: AtomicU32, #[pin] work: Work, #[pin] @@ -795,6 +877,8 @@ struct Shared { state: Mutex>>, #[pin] changed: CondVar, + /// The port of the session in `state`; null before boot and once detachment begins. + port: core::sync::atomic::AtomicPtr, } impl_has_work! { @@ -884,6 +968,27 @@ impl Shared { drop(retired); } + /// Wakes the waiters for a returned pair, binding or capacity change. + fn note_changed(&self) { + self.changed.notify_all(); + self.renders.notify(); + } + + + /// The session's port, valid while `lease` is held: shutdown and device loss join every + /// lease before they detach the session. + fn port<'a>(&self, lease: &'a preparation::Lease) -> Option<&'a port::Port> { + let _ = lease; + // SAFETY: The pointer is published before the session becomes reachable and cleared + // only after every lease has been joined, so it names a live port for 'a. + unsafe { self.port.load(Ordering::Acquire).as_ref() } + } + + /// Called after joining every preparation lease, before the session is detached. + fn forget_port(&self) { + self.port.store(core::ptr::null_mut(), Ordering::Release); + } + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { // Already queued if this fails, which is just as good. @@ -912,6 +1017,7 @@ impl Shared { // Context release returns its execution root off-lock. The next // device pass can now detach idle compute aliases and their QoS // leases, without depending on a new client's submission/event. + self.queues.notify(); self.queue_events(); } } @@ -1023,7 +1129,11 @@ impl Gpu { oas: cfg.uat_oas, }; let max_frequency_khz = Self::max_frequency_khz(dev)?; - let preparations = preparation::Gate::new()?; + let channels = availability::Availability::new()?; + let progress = availability::Availability::new()?; + let queues = availability::Availability::new()?; + let preparations = preparation::Gate::with_waiters(Some(progress.clone()))?; + let renders = availability::Availability::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); @@ -1054,6 +1164,11 @@ impl Gpu { feed, feed_work <- new_work!("g17::Shared::feed_work"), preparations, + channels, + progress, + queues, + renders, + compute_demand: AtomicU32::new(0), work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), idle <- new_delayed_work!("g17::Shared::idle"), @@ -1063,6 +1178,7 @@ impl Gpu { stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), + port: core::sync::atomic::AtomicPtr::new(core::ptr::null_mut()), }), GFP_KERNEL, )?; @@ -1085,6 +1201,9 @@ impl Gpu { GFP_KERNEL, )?; let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + gpu.shared + .port + .store(Arc::as_ptr(&firmware.port).cast_mut(), Ordering::Release); *gpu.shared.state.lock() = Some(firmware); gpu.shared.queue_events(); Ok(gpu) @@ -1215,17 +1334,21 @@ impl Gpu { secondary, primary, bridge, + port: port::Port::new()?, + seen_submissions: 0, rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), hold: power::Hold::new(), registration_power_release_pending: false, variant: shared.variant, - queues <- runtime::Registry::new( + queues <- runtime::Registry::with_waiters( shared.clusters, shared.deflake, shared.descriptor_flags, shared.variant, + Some(shared.queues.clone()), + Some(shared.renders.clone()), ), pool: object::Pool::new()?, render_rings: kick::RenderRings::new()?, @@ -1261,6 +1384,8 @@ impl Gpu { if let Some(root_entries) = root_entries { session.uat.confirm_firmware_root_entries(root_entries)?; } + let mailbox: *const Mailbox = &*session.primary.rtkit; + session.port.attach(session.init.port_views(mailbox)?, session.bridge.regs.clone())?; session.uat.mirror_secondary_root()?; if cfg.fabric_bridge { session.bridge.regs.enable_g17_gating()?; @@ -1396,7 +1521,8 @@ impl gpu::Gpu for Gpu { u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS) | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS) - | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD); + | u64::from(uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD) + | sync_output::FEATURE_CUMULATIVE_SIGNALS; let variant = self.shared.variant; if variant.few_primitives { params.features |= @@ -1494,16 +1620,21 @@ impl gpu::Gpu for Gpu { let policy = scheduling_policy(priority)?; let admission_started = now_ns(); let (backend, slot) = loop { + let observed = self.shared.queues.observe(); match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, Err(error) if error == ENOSPC => { // Closed logical queues can still own execution/QoS IDs // until the scheduler release is consumed and reclaimed. // Backend construction has rolled back all temporary leases. - if now_ns().saturating_sub(admission_started) - >= job::TIMEOUT_MS as u64 * 1_000_000 { return Err(EBUSY); } + let waited = now_ns().saturating_sub(admission_started); + let limit = job::TIMEOUT_MS as u64 * 1_000_000; + if waited >= limit { return Err(EBUSY); } self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A returned queue slot or reclaimed context wakes this. + self.shared + .queues + .block(observed, (limit - waited).div_ceil(1_000_000) as u32, true)?; } result => break result?, } @@ -1529,18 +1660,22 @@ impl gpu::Gpu for Gpu { mut bo: gem::ObjectRef, range: Range, ) -> Result { - let state = self.shared.state.lock(); - let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - crate::timestamp::Buffer::firmware(bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - addresses, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - )?) + let mapping = { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + )? + }; + // The buffer's CPU mapping takes the reservation lock: not under the device mutex. + crate::timestamp::Buffer::firmware(mapping) } } diff --git a/drivers/gpu/drm/asahi/g17/port.rs b/drivers/gpu/drm/asahi/g17/port.rs new file mode 100644 index 0000000000000..db043ef9d6f91 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/port.rs @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-wide publication state that physical queues share without the device mutex. +//! +//! A firmware session owns its port, and the views it holds name that session's +//! firmware-shared memory. Code that does not hold the device mutex reaches the port only +//! through a preparation lease ([`super::Shared::port`]): shutdown and device loss join every +//! lease before they detach the session. Each piece below is atomic or has its own lock. +//! +//! Lock order: device mutex -> compute queue cell -> the locks of this port. The render +//! publication lock comes first among the port's locks: device mutex -> render publication +//! lock -> admission pool, QoS, mailbox. Its holder never takes the device mutex. + +use super::{ + fw::{ + channels::{WorkSlot, WORK_RING_SLOTS}, + queue::{DataMaster, Policy}, + }, + qos, +}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicU64, Ordering}; +use kernel::{ + new_mutex, new_spinlock, + prelude::*, + sync::{Arc, Mutex, MutexGuard, SetOnce, SpinLock}, +}; + +const PRIORITIES: usize = Policy::PRIORITIES as usize; + +/// One priority class's compute work ring: its slot array and the cursors of its two firmware +/// consumers and of the host producer. +#[derive(Clone, Copy)] +pub(super) struct RingView { + pub(super) slots: *mut u8, + pub(super) consumers: [*const AtomicU32; 2], + pub(super) producer: *const AtomicU32, +} + +impl RingView { + fn cursors(&self) -> Result<[u32; 3]> { + // SAFETY: The views name 32-bit words of the session's retained firmware objects, which + // the port's owner keeps alive. Other agents only use single 32-bit accesses. + let values = unsafe { + [ + (*self.consumers[0]).load(Ordering::Relaxed), + (*self.consumers[1]).load(Ordering::Relaxed), + (*self.producer).load(Ordering::Relaxed), + ] + }; + if values.iter().any(|&value| value >= WORK_RING_SLOTS) { + return Err(EINVAL); + } + Ok(values) + } + + /// Slots the host may still fill before reaching either consumer. + fn free(&self) -> Result { + let [first, second, producer] = self.cursors()?; + let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + Ok(room(first).min(room(second))) + } +} + +/// Raw views of the session's firmware-shared memory, fixed once the firmware is built. +pub(super) struct Views { + pub(super) compute: [RingView; PRIORITIES], + pub(super) tiling: [RingView; PRIORITIES], + pub(super) fragment: [RingView; PRIORITIES], + /// Completion stamp page A: one word per QID, written by the firmware. + pub(super) stamps: *const AtomicU32, + pub(super) qos: *mut u8, + pub(super) recovery: *const AtomicU32, + pub(super) mailbox: *const super::Mailbox, +} + +// SAFETY: The views only name memory and a mailbox owned by the firmware session, which outlives +// every user of the port. Accesses are atomic words or serialized by the port's locks. +unsafe impl Send for Views {} +// SAFETY: As above. +unsafe impl Sync for Views {} + +#[pin_data] +pub(super) struct Port { + views: SetOnce, + /// Register access for tracing the core power state at the doorbell. + regs: SetOnce, + /// Compute work-ring slots reserved by publishers that have not exposed them yet. + #[pin] + rings: SpinLock<[u32; PRIORITIES]>, + #[pin] + qos: SpinLock, + /// Serializes every render publication, with or without the device mutex: the tiling + /// and fragment work rings have one host producer at a time, and same-VM attachment + /// overlaps are ordered by publication order (see hazard.rs). + #[pin] + render: Mutex<()>, + /// The accepted recovery generation, mirrored for publishers without the device mutex. + generation: AtomicU64, + /// Device publication order shared by compute and render commands. + publications: AtomicU64, + /// Compute work slots written per priority class. + compute_written: [AtomicU64; PRIORITIES], + /// `compute_written` as read before the latest compute doorbell of each class: every slot + /// counted there was in its ring when the firmware handled that doorbell. + compute_announced: [AtomicU64; PRIORITIES], + /// Full firmware effort is applied; cleared before an idle clear rechecks `submissions`. + effort_full: AtomicBool, + submissions: AtomicU64, +} + +/// Room in both render work rings of one priority, observed under the render publication +/// lock. While that lock is held no other host producer moves, and the firmware consumers only +/// advance, so the observed producers stay current and the room stays available. +#[derive(Clone, Copy)] +pub(super) struct RenderRoom { + priority: u8, + /// Fragment, then tiling producer. + producers: [u32; 2], +} + +impl RenderRoom { + pub(super) fn priority(&self) -> u8 { + self.priority + } +} + +/// A compute work-ring slot reserved before any command becomes visible. Dropping an unused +/// claim returns the slot. +pub(super) struct OuterClaim<'a> { + port: &'a Port, + priority: usize, + armed: bool, +} + +impl Drop for OuterClaim<'_> { + fn drop(&mut self) { + if self.armed { + let mut reserved = self.port.rings.lock(); + reserved[self.priority] = reserved[self.priority].saturating_sub(1); + } + } +} + +impl Port { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + views: SetOnce::new(), + regs: SetOnce::new(), + rings <- new_spinlock!([0; PRIORITIES], "g17::Port::rings"), + qos <- new_spinlock!(qos::Accounting::new(), "g17::Port::qos"), + render <- new_mutex!((), "g17::Port::render"), + generation: AtomicU64::new(0), + publications: AtomicU64::new(0), + compute_written: [const { AtomicU64::new(0) }; PRIORITIES], + compute_announced: [const { AtomicU64::new(0) }; PRIORITIES], + effort_full: AtomicBool::new(false), + submissions: AtomicU64::new(0), + }), + GFP_KERNEL, + ) + } + + /// Called once while the session is built, before any publisher can see the port. + pub(super) fn attach(&self, views: Views, regs: crate::regs::G17Resources) -> Result { + if self.views.populate(views) && self.regs.populate(regs) { + Ok(()) + } else { + Err(EBUSY) + } + } + + fn views(&self) -> Result<&Views> { + self.views.as_ref().ok_or(ENODEV) + } + + fn ring(&self, priority: u8) -> Result<(usize, RingView)> { + let index = usize::from(priority); + Ok((index, *self.views()?.compute.get(index).ok_or(EINVAL)?)) + } + + /// Reserves one slot of the compute work ring of `priority`. EAGAIN while firmware has not + /// consumed enough slots; only consumption, never a waiting publisher, frees one. + pub(super) fn reserve_compute(&self, priority: u8) -> Result> { + let (index, ring) = self.ring(priority)?; + let mut reserved = self.rings.lock(); + if ring.free()? <= reserved[index] { + return Err(EAGAIN); + } + reserved[index] += 1; + Ok(OuterClaim { + port: self, + priority: index, + armed: true, + }) + } + + /// Whether a compute publication at `priority` can reserve a slot now. + pub(super) fn compute_ready(&self, priority: u8) -> Result { + let (index, ring) = self.ring(priority)?; + let reserved = self.rings.lock(); + Ok(ring.free()? > reserved[index]) + } + + /// Exposes `record` in its reserved slot. Inner queue records must be visible first. Returns + /// the new producer, the retained consumer witness for this slot. + pub(super) fn publish_compute(&self, mut claim: OuterClaim<'_>, record: &WorkSlot) -> Result { + if record.data_master != DataMaster::Compute as u32 || !core::ptr::eq(claim.port, self) { + return Err(EINVAL); + } + let ring = self.views()?.compute[claim.priority]; + let mut reserved = self.rings.lock(); + let [first, second, producer] = ring.cursors()?; + let next = (producer + 1) % WORK_RING_SLOTS; + if next == first || next == second { + return Err(EIO); + } + // SAFETY: The ring view spans WORK_RING_SLOTS records, checked when the session built + // it; `producer` is in range. The reservation proves firmware does not own this slot and + // the ring lock excludes every other host writer. The record is a padding-free ABI value. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref(record).cast::(), + ring.slots.add(producer as usize * core::mem::size_of::()), + core::mem::size_of::(), + ); + } + fence(Ordering::SeqCst); + // SAFETY: As for `RingView::cursors`. + unsafe { (*ring.producer).store(next, Ordering::Relaxed) }; + fence(Ordering::SeqCst); + reserved[claim.priority] = reserved[claim.priority].saturating_sub(1); + self.compute_written[claim.priority].fetch_add(1, Ordering::AcqRel); + claim.armed = false; + Ok(next) + } + + /// Compute work slots written at `priority` so far. + pub(super) fn compute_written(&self, priority: u8) -> u64 { + self.compute_written + .get(usize::from(priority)) + .map_or(u64::MAX, |written| written.load(Ordering::Acquire)) + } + + /// Compute work slots at `priority` that a doorbell has announced. + pub(super) fn compute_announced(&self, priority: u8) -> u64 { + self.compute_announced + .get(usize::from(priority)) + .map_or(0, |announced| announced.load(Ordering::Acquire)) + } + + /// Rings the compute doorbell `message` of `priority`, after noting which slots it + /// announces. + pub(super) fn notify_compute(&self, priority: u8, message: u64) -> Result { + let index = usize::from(priority); + let written = self.compute_written.get(index).ok_or(EINVAL)?.load(Ordering::Acquire); + self.notify(message)?; + self.compute_announced[index].fetch_max(written, Ordering::AcqRel); + Ok(()) + } + + /// Taken around every render publication; see `render`. + pub(super) fn render_lock(&self) -> MutexGuard<'_, ()> { + self.render.lock() + } + + fn render_rings(&self, priority: u8) -> Result<(RingView, RingView)> { + let views = self.views()?; + let index = usize::from(priority); + Ok(( + *views.fragment.get(index).ok_or(EINVAL)?, + *views.tiling.get(index).ok_or(EINVAL)?, + )) + } + + /// Reads both render work rings of `priority` once. Returns their producers if both have a + /// free slot; the room holds for as long as `guard`. Only firmware consumption, never a + /// publisher without the render lock, changes it. + pub(super) fn observe_render_room( + &self, + _guard: &MutexGuard<'_, ()>, + priority: u8, + ) -> Result> { + let (fragment, tiling) = self.render_rings(priority)?; + let cursors = [fragment.cursors()?, tiling.cursors()?]; + let room = |[first, second, producer]: [u32; 3]| { + let room = |consumer: u32| (consumer + WORK_RING_SLOTS - producer - 1) % WORK_RING_SLOTS; + room(first).min(room(second)) + }; + Ok((room(cursors[0]) > 0 && room(cursors[1]) > 0).then_some(RenderRoom { + priority, + producers: [cursors[0][2], cursors[1][2]], + })) + } + + /// Copies a render pair's fragment, then tiling record into the slots after the producers + /// that `room` observed, without exposing them, as `InitData::publish_work_pair` does for a + /// deferred pair. Returns the producers that expose them. + pub(super) fn write_render_pair( + &self, + _guard: &MutexGuard<'_, ()>, + room: &RenderRoom, + fragment: &WorkSlot, + tiling: &WorkSlot, + commit_accounting: impl FnOnce() -> Result, + publish_late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let (fragment_ring, tiling_ring) = self.render_rings(room.priority)?; + let slot = |ring: &RingView, producer: u32| -> Result<(*mut u8, u32)> { + if producer >= WORK_RING_SLOTS { + return Err(EINVAL); + } + // SAFETY: The ring view spans WORK_RING_SLOTS records; `producer` is in range. + let slot = unsafe { ring.slots.add(producer as usize * core::mem::size_of::()) }; + Ok((slot, (producer + 1) % WORK_RING_SLOTS)) + }; + let (fragment_slot, fragment_next) = slot(&fragment_ring, room.producers[0])?; + let (tiling_slot, tiling_next) = slot(&tiling_ring, room.producers[1])?; + let copy = |record: &WorkSlot, destination: *mut u8| { + // SAFETY: The room proves that the slot after the producer is host-owned until the + // producer moves past it, and the render lock excludes every other host writer. + unsafe { + core::ptr::copy_nonoverlapping( + core::ptr::from_ref(record).cast::(), + destination, + core::mem::size_of::(), + ) + } + }; + copy(fragment, fragment_slot); + commit_accounting()?; + fence(Ordering::SeqCst); + publish_late_tiling(); + copy(tiling, tiling_slot); + fence(Ordering::SeqCst); + Ok([fragment_next, tiling_next]) + } + + /// Exposes a pair written by `write_render_pair` with `room`, fragment first, as + /// `InitData::publish_retained_work_pair` does: `next` must follow the observed producers. + pub(super) fn expose_render_pair( + &self, + _guard: &MutexGuard<'_, ()>, + room: &RenderRoom, + next: [u32; 2], + ) -> Result { + let (fragment_ring, tiling_ring) = self.render_rings(room.priority)?; + for (producer, next) in room.producers.into_iter().zip(next) { + if next != (producer + 1) % WORK_RING_SLOTS { + return Err(EBUSY); + } + } + fence(Ordering::SeqCst); + // SAFETY: As for `RingView::cursors`. + unsafe { (*fragment_ring.producer).store(next[0], Ordering::Relaxed) }; + fence(Ordering::SeqCst); + // SAFETY: As above. + unsafe { (*tiling_ring.producer).store(next[1], Ordering::Relaxed) }; + fence(Ordering::SeqCst); + Ok(()) + } + + /// The completion stamp of `qid` in page A. + pub(super) fn stamp(&self, qid: u8) -> Result { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let base = self.views()?.stamps; + // SAFETY: Page A holds one word per QID up to QID_MAX; the session retains it. + let value = unsafe { (*base.add(usize::from(qid))).load(Ordering::Relaxed) }; + fence(Ordering::Acquire); + Ok(value) + } + + fn qos_view(&self) -> Result> { + // SAFETY: The session retains the QoS table for at least as long as this port. + unsafe { qos::View::from_raw(self.views()?.qos) } + } + + pub(super) fn qos_publish( + &self, + owner: qos::Owner, + scheduler: u64, + policy: Policy, + ) -> Result { + let view = self.qos_view()?; + self.qos.lock().publish(&view, owner, scheduler, policy, qos::clock) + } + + /// Publishes the two owners of a render command, fragment then tiling. + pub(super) fn qos_publish_pair( + &self, + owners: [qos::Owner; 2], + scheduler: u64, + policy: Policy, + ) -> Result<[qos::Publication; 2]> { + let view = self.qos_view()?; + self.qos.lock().publish_owners(&view, owners, scheduler, policy, qos::clock) + } + + pub(super) fn qos_cancel(&self, publication: qos::Publication) -> Result { + let view = self.qos_view()?; + self.qos.lock().cancel(&view, publication) + } + + pub(super) fn qos_complete(&self, owners: &[qos::Owner]) -> Result { + let view = self.qos_view()?; + self.qos.lock().complete(&view, owners) + } + + /// Rings the primary coprocessor's doorbell. + pub(super) fn notify(&self, message: u64) -> Result { + // SAFETY: The session's primary coprocessor owns the mailbox and outlives the port. + let mailbox = unsafe { &*self.views()?.mailbox }; + super::trace::doorbell(super::trace::COMPUTE, || { + self.regs.as_ref().map_or(0, |regs| regs.g17_core_power()) + }); + super::trace::message(super::trace::MAILBOX, message); + mailbox.send(super::EP_DOORBELL, message) + } + + pub(super) fn recovery_state(&self) -> Result { + let word = self.views()?.recovery; + fence(Ordering::Acquire); + // SAFETY: As for `RingView::cursors`. + Ok(unsafe { (*word).load(Ordering::Relaxed) }) + } + + pub(super) fn generation(&self) -> u64 { + self.generation.load(Ordering::Acquire) + } + + /// Called with the device mutex held after the accepted generation changed, before + /// publication reopens. + pub(super) fn set_generation(&self, generation: u64) { + self.generation.store(generation, Ordering::Release); + } + + /// Unique reservation order shared by both engines. Concurrent publishers may expose + /// their work in a different order; queue serialization and visibility use other locks. + /// The watchdog uses this number only to choose bounded cross-VM timeout grace. + pub(super) fn next_publication(&self) -> u64 { + self.publications.fetch_add(1, Ordering::Relaxed) + 1 + } + + /// A publisher without the device mutex counts itself before it checks that full effort is + /// applied. The idle clear withdraws the flag before it rechecks the count, so either the + /// clear sees this submission or this submission sees the flag withdrawn. + pub(super) fn note_submission(&self) -> bool { + self.submissions.fetch_add(1, Ordering::SeqCst); + self.effort_full.load(Ordering::SeqCst) + } + + pub(super) fn submissions(&self) -> u64 { + self.submissions.load(Ordering::SeqCst) + } + + pub(super) fn set_effort_full(&self, full: bool) { + self.effort_full.store(full, Ordering::SeqCst); + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 7fef15084c50a..b0e21e70bbe18 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -6,7 +6,7 @@ use core::sync::atomic::AtomicU32; pub(super) const FULL_EFFORT: u32 = 0x10000; -const IDLE_HOLD_NS: u64 = 16_000_000; +pub(super) const IDLE_HOLD_NS: u64 = 16_000_000; /// The firmware's idle power-off delay as measured on T6040/25G83: the cores power off 2 ms after /// the last job. Not measured on other firmware; at or below it the hold stays off. diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs index 0297c619b4f93..eb2366fa3451b 100644 --- a/drivers/gpu/drm/asahi/g17/preparation.rs +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -55,20 +55,37 @@ pub(super) struct Gate { wait: Mutex<()>, #[pin] changed: CondVar, + /// Scheduler admission waiting for publication to reopen. + available: Option>, } impl Gate { + #[cfg(CONFIG_KUNIT)] pub(super) fn new() -> Result> { + Self::with_waiters(None) + } + + /// A gate whose reopening, failure and removal also wake `available`. + pub(super) fn with_waiters( + available: Option>, + ) -> Result> { Arc::pin_init( pin_init!(Self { state: AtomicU64::new(EPOCH), wait <- new_mutex!((), "g17::Gate::wait"), changed <- new_condvar!("g17::Gate::changed"), + available, }), GFP_KERNEL, ) } + fn wake_waiters(&self) { + if let Some(available) = &self.available { + available.notify(); + } + } + pub(super) fn try_enter(self: &Arc) -> Result { let mut previous = self.state.load(Ordering::Acquire); loop { @@ -162,20 +179,55 @@ impl Gate { } } + /// Also wakes publishers in wait_closure(); they then block in enter(). pub(super) fn close(&self) { self.invalidate(0); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + /// Opaque snapshot for wait_closure(), taken where recovery was observed. + pub(super) fn snapshot(&self) -> u64 { + self.state.load(Ordering::Acquire) + } + + /// A publisher that observed recovery (host pending, or the firmware's state + /// ahead of the event worker, which always follows with a restart and the + /// closure) sleeps until this gate has closed since `seen`. enter() then + /// blocks until recovery reopens it. The deadline is only a failure bound. + pub(super) fn wait_closure(&self, seen: u64, deadline: u64) -> Result { + let mut wait = self.wait.lock(); + loop { + let state = self.state.load(Ordering::Acquire); + if state & (REMOVED | FAILED) != 0 { + return Err(ENODEV); + } + if state & CLOSED != 0 || state & !COUNT_MASK != seen & !COUNT_MASK { + return Ok(()); + } + let remaining = deadline.saturating_sub(super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + let ms = u32::try_from(remaining.div_ceil(1_000_000)).unwrap_or(u32::MAX); + let _ = self.changed.wait_timeout(&mut wait, kernel::time::msecs_to_jiffies(ms)); + } } pub(super) fn remove(&self) { self.invalidate(REMOVED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } pub(super) fn fail(&self) { self.invalidate(FAILED); - let _wait = self.wait.lock(); + let wait = self.wait.lock(); self.changed.notify_all(); + drop(wait); + self.wake_waiters(); } /// Runtime failure/recovery must not join an indefinitely stalled builder. @@ -199,7 +251,7 @@ impl Gate { /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. pub(super) fn reopen(&self) -> bool { - let _wait = self.wait.lock(); + let wait = self.wait.lock(); let mut previous = self.state.load(Ordering::Acquire); loop { if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { @@ -213,6 +265,8 @@ impl Gate { ) { Ok(_) => { self.changed.notify_all(); + drop(wait); + self.wake_waiters(); return true; } Err(current) => previous = current, @@ -307,9 +361,9 @@ mod compute_deadline_tests { #[test] fn recovery_and_binding_retry_churn_cannot_renew_budget() { let gate = Gate::new().unwrap(); - let start = super::super::now_ns(); + let deadline = super::super::now_ns() + 20_000_000; let check = || Ok(()); - let admission = Admission::new(start + 20_000_000, &check); + let admission = Admission::new(deadline, &check); loop { // Re-entry models a channel taken between admission/publication. // Recovery joins each dropped lease, then wakes the same budget. @@ -317,12 +371,17 @@ mod compute_deadline_tests { gate.changed.notify_all(); gate.wait_drained(); assert!(gate.reopen()); + // However often the gate reopened, an attempt that starts at the deadline + // is refused. Timing the whole loop would also count time descheduled. + let attempted = super::super::now_ns(); match gate.enter_until(&admission) { - Ok(lease) => drop(lease), + Ok(lease) => { + assert!(attempted < deadline); + drop(lease); + } Err(error) => { assert_eq!(error, EBUSY); break; } } } - assert!(super::super::now_ns() - start < 500_000_000); assert_eq!(gate.state.load(Ordering::Acquire) & COUNT_MASK, 0); } diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index 9a100db4cffdf..b36656b0174a7 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -2,17 +2,14 @@ //! Shared QoS publication and exact-completion accounting for every engine. //! -//! The device publication lock serializes this host state. Publishing identity +//! The port's QoS lock serializes this host state. Publishing identity //! precedes submitted counters; retiring a final owner clears only its two row //! bytes. Cancellation balances host references without rewinding shared counts. -use super::{ - fw::{qos as fw, queue::Policy}, - object::KernelObject, -}; +use super::fw::{qos as fw, queue::Policy}; use crate::hw::t8140::qos as cfg; use core::{ - mem::{offset_of, size_of}, + mem::offset_of, sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, }; use kernel::prelude::*; @@ -30,15 +27,17 @@ pub(crate) struct View<'a> { times_and_counts: &'a [AtomicU64; 7], } impl<'a> View<'a> { - pub(crate) fn new(object: &'a KernelObject) -> Result { - let ptr = object.pointer(0, size_of::())?; + /// # Safety + /// + /// `ptr` must span a complete QoS table that stays mapped for `'a`. + pub(crate) unsafe fn from_raw(ptr: *mut u8) -> Result { if ptr as usize & 7 != 0 { return Err(EINVAL); } // SAFETY: pointer spans the complete layout, whose asserted offsets // and size bound every array below. Integer atomics have their integer - // representation and alignment. The retained object borrow keeps every - // shared reference alive; accesses use each field's fixed width. + // representation and alignment. The caller keeps every shared + // reference alive; accesses use each field's fixed width. Ok(unsafe { Self { rows: &*ptr.add(offset_of!(fw::Qos, rows)).cast(), @@ -54,6 +53,14 @@ impl<'a> View<'a> { fn row(&self, qid: usize) -> [u8; 8] { core::array::from_fn(|index| self.rows[qid][index].load(Ordering::Relaxed)) } + /// The complete row in one access. The table is uncached: a publication reads + /// each row once, before it stores anything. + fn row_word(&self, qid: usize) -> [u8; 8] { + let row = core::ptr::from_ref(&self.rows[qid]).cast::(); + // SAFETY: Rows are eight bytes at eight-byte offsets of the eight-byte + // aligned table (checked in from_raw), so the row is one aligned word. + unsafe { (*row).load(Ordering::Relaxed) }.to_le_bytes() + } } #[derive(Clone, Copy, Debug)] @@ -109,66 +116,111 @@ impl Accounting { policy: Policy, clock: impl FnOnce() -> (u64, u64), ) -> Result { - let (qid, qos, dm) = owner.indices()?; + let [publication] = self.publish_owners(view, [owner], scheduler, policy, clock)?; + Ok(publication) + } + + /// Publishes the owners of one command's queues, in order, with the same + /// firmware-visible values as one `publish` per owner. The owners use + /// distinct queues and data masters. Every check precedes the first store, + /// so a refusal changes nothing; every table word is read before the first + /// store, so no uncached read waits for an earlier store to drain. + pub(crate) fn publish_owners( + &mut self, + view: &View<'_>, + owners: [Owner; N], + scheduler: u64, + policy: Policy, + clock: impl FnOnce() -> (u64, u64), + ) -> Result<[Publication; N]> { if scheduler == 0 { return Err(EINVAL); } - let next_qid = self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; - let next_dm = self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; - let starts_busy = self.dm_in_flight[dm] == 0; - let time = if starts_busy { + let mut indices = [(0, 0, 0); N]; + for (index, owner) in owners.iter().enumerate() { + indices[index] = owner.indices()?; + let (qid, _, dm) = indices[index]; + if indices[..index] + .iter() + .any(|&(other_qid, _, other_dm)| other_qid == qid || other_dm == dm) + { + return Err(EINVAL); + } + self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; + self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; + } + let rows = indices.map(|(qid, _, _)| view.row_word(qid)); + let queue_submitted = indices.map(|(qid, _, _)| view.queue_submitted[qid].load(Ordering::Relaxed)); + let owner_submitted = indices.map(|(_, qos, _)| view.owner_submitted[qos].load(Ordering::Relaxed)); + let dm_submitted = indices.map(|(_, _, dm)| view.times_and_counts[3 + dm].load(Ordering::Relaxed)); + let starts_busy = indices.map(|(_, _, dm)| self.dm_in_flight[dm] == 0); + let busy = starts_busy.iter().any(|&starts| starts); + let intervals = if busy { view.intervals.load(Ordering::Relaxed) } else { 0 }; + for (index, owner) in owners.iter().enumerate() { + let (qid, _, _) = indices[index]; + if self.in_flight[qid] != 0 + && (rows[index][0] != owner.qos || self.dm_by_qid[qid] != owner.data_master) + { + return Err(EBUSY); + } + } + let time = if busy { let (counter, frequency) = clock(); time_from_host(counter, frequency)? } else { 0 }; - if self.in_flight[qid] != 0 - && (view.rows[qid][0].load(Ordering::Relaxed) != owner.qos - || self.dm_by_qid[qid] != owner.data_master) - { - return Err(EBUSY); + let class = policy.qos_class(); + let mut first = false; + for (index, owner) in owners.iter().enumerate() { + let (qid, qos, _) = indices[index]; + view.rows[qid][0].store(owner.qos, Ordering::Relaxed); + view.rows[qid][1].store(class, Ordering::Relaxed); + view.scheduler[qos].store(scheduler, Ordering::Relaxed); + view.share[qos].store(policy.qos_share(), Ordering::Relaxed); + first |= self.in_flight[qid] == 0; } - view.rows[qid][0].store(owner.qos, Ordering::Relaxed); - view.rows[qid][1].store(policy.qos_class(), Ordering::Relaxed); - let record = view.row(qid); - view.scheduler[qos].store(scheduler, Ordering::Relaxed); - view.share[qos].store(policy.qos_share(), Ordering::Relaxed); - if self.in_flight[qid] == 0 { + if first { fence(Ordering::SeqCst); } - view.queue_submitted[qid].store( - view.queue_submitted[qid] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - view.owner_submitted[qos].store( - view.owner_submitted[qos] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - if starts_busy { - view.times_and_counts[dm].store(time, Ordering::Relaxed); - } - let submitted = &view.times_and_counts[3 + dm]; - let dm_submitted = submitted.load(Ordering::Relaxed).wrapping_add(1); - submitted.store(dm_submitted, Ordering::Relaxed); - if starts_busy { - view.intervals.store( - view.intervals.load(Ordering::Relaxed).wrapping_add(1), + let mut publications = [Publication { + record: [0; 8], + owner: owners[0], + dm_submitted: 0, + }; N]; + let mut intervals = intervals; + for (index, owner) in owners.iter().enumerate() { + let (qid, qos, dm) = indices[index]; + view.queue_submitted[qid].store(queue_submitted[index].wrapping_add(1), Ordering::Relaxed); + // Owners of one logical queue share their submission count. + let earlier = indices[..index].iter().filter(|&&(_, other, _)| other == qos).count(); + view.owner_submitted[qos].store( + owner_submitted[index].wrapping_add(earlier as u32 + 1), Ordering::Relaxed, ); + if starts_busy[index] { + view.times_and_counts[dm].store(time, Ordering::Relaxed); + } + let submitted = dm_submitted[index].wrapping_add(1); + view.times_and_counts[3 + dm].store(submitted, Ordering::Relaxed); + if starts_busy[index] { + intervals = intervals.wrapping_add(1); + view.intervals.store(intervals, Ordering::Relaxed); + } + let mut record = rows[index]; + record[0] = owner.qos; + record[1] = class; + self.in_flight[qid] += 1; + self.dm_in_flight[dm] += 1; + self.dm_by_qid[qid] = owner.data_master; + publications[index] = Publication { + record, + owner: *owner, + dm_submitted: submitted, + }; } - self.in_flight[qid] = next_qid; - self.dm_in_flight[dm] = next_dm; - self.dm_by_qid[qid] = owner.data_master; fence(Ordering::SeqCst); - Ok(Publication { - record, - owner, - dm_submitted, - }) + Ok(publications) } /// A possibly exposed producer must retain its reference until completion. diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs index ebdc79d2b2dd3..e1e7552c243f2 100644 --- a/drivers/gpu/drm/asahi/g17/queue/admission.rs +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -9,6 +9,7 @@ //! a graph or waiting for GPU work. Capacity waiters sleep on the release event. use crate::g17::{ + fw::kick::KickDependency, hazard::{self, Footprint, Plan}, job::Packet, }; @@ -43,6 +44,9 @@ struct InFlight { slot: Option, ticket: u64, footprint: Arc, + /// Fragment kick of the published render; a later same-VM overlap of + /// another pair waits for it in firmware. + fragment: Option, } const QUARANTINE_WAIT_MS: u32 = 1000; @@ -294,6 +298,28 @@ impl State { ) } + /// Whether `pick` would return a slot, without taking it. + fn can_pick(&self, owner: u64, plan: Plan) -> bool { + let usable = |slot: usize| { + self.owners[slot] == owner + && (self.quarantined | self.kill_hold) & (1u64 << slot) == 0 + && self.refs[slot] < cfg::RENDER_DEPTH + }; + match plan { + Plan::Wait => false, + Plan::Pair(slot) => usize::from(slot) < cfg::RENDER_SLOTS && usable(usize::from(slot)), + Plan::Any if owner == 0 => false, + Plan::Any if self.owners.contains(&owner) => (0..cfg::RENDER_SLOTS).any(usable), + Plan::Any => { + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + (0..cfg::RENDER_SLOTS).any(|slot| { + free & (1u64 << slot) != 0 + && (self.released & (1u64 << slot) != 0 || self.owners[slot] == 0) + }) + } + } + } + fn pick(&mut self, owner: u64, plan: Plan) -> Option { match plan { Plan::Any => self.acquire(owner), @@ -332,6 +358,7 @@ impl State { slot, ticket, footprint: request.footprint.clone(), + fragment: None, }, GFP_KERNEL, ) @@ -339,6 +366,12 @@ impl State { Some((owner, request.sequence)) } + fn position(&self, slot: u8, (owner, sequence): (u64, u64)) -> Option { + self.footprints.iter().position(|entry| { + entry.slot == Some(slot) && entry.owner == owner && entry.sequence == sequence + }) + } + fn forget_waiter(&mut self, ticket: Option) { if let Some(ticket) = ticket { self.footprints @@ -366,9 +399,22 @@ pub(crate) struct Pool { queues: AtomicU32, /// Logical queues admitted at most: each must be able to hold a render pair. max_queues: u32, + /// Logical-queue creation waiting for a returned slot. + waiters: Option>, + /// Render publication waiting for render capacity: woken with every + /// release event above. + renders: Option>, } impl Pool { + #[cfg(CONFIG_KUNIT)] pub(crate) fn new(max_queues: u32) -> Result> { + Self::with_waiters(max_queues, None, None) + } + pub(crate) fn with_waiters( + max_queues: u32, + waiters: Option>, + renders: Option>, + ) -> Result> { if max_queues > MAX_EXECUTION_CONTEXTS { return Err(EINVAL); } @@ -378,10 +424,19 @@ impl Pool { released <- new_condvar!("G17 render pair release"), queues: AtomicU32::new(0), max_queues, + waiters, + renders, }), GFP_KERNEL, ) } + /// Every pair or lease return, owner close and quarantine change. + fn returned(&self) { + self.released.notify_all(); + if let Some(renders) = &self.renders { + renders.notify(); + } + } /// Logical queues currently counted, including one being created. pub(crate) fn queues(&self) -> u32 { self.queues.load(Ordering::Acquire) @@ -522,6 +577,24 @@ impl Pool { record, })) } + /// Whether `acquire_render` would admit this request now, without + /// admitting it: the scheduler's prepare step waits for this. + pub(crate) fn admits_render(&self, owner: u64, request: Option>) -> bool { + let state = self.state.lock(); + if owner == 0 || state.refusal(owner).is_err() { + return false; + } + if state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) + { + return false; + } + let plan = state.plan(owner, request, None); + state.can_pick(owner, plan) + } pub(crate) fn warmable_sibling( self: &Arc, owner: u64, @@ -558,7 +631,7 @@ impl Pool { } } drop(state); - self.released.notify_all(); + self.returned(); } /// Active leases survive this transition. Their later drop is still the /// event which releases capacity, so a stopped owner is never double-freed. @@ -576,7 +649,7 @@ impl Pool { state.spared &= !released; state.kill_hold = 0; drop(state); - self.released.notify_all(); + self.returned(); } } @@ -653,9 +726,18 @@ mod accounting_tests { pub(crate) struct QueueSlot { pool: Arc, } +impl QueueSlot { + /// The device's admission pool, which lives as long as the firmware session. + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } +} impl Drop for QueueSlot { fn drop(&mut self) { self.pool.queues.fetch_sub(1, Ordering::AcqRel); + if let Some(waiters) = &self.pool.waiters { + waiters.notify(); + } } } @@ -691,7 +773,7 @@ impl RenderLease { } state.quarantined |= bit; drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } /// The VM pin must be retired first. Keep this lease until after clearing /// quarantine; its drop is what permits waiting publications to proceed. @@ -705,19 +787,78 @@ impl RenderLease { self.rehabilitate_after_processor_stop(); } } +impl RenderLease { + /// Called under the device mutex once the render's kicks are committed, + /// including a replay, which supersedes the earlier kick. + pub(crate) fn note_publication(&self, fragment: KickDependency) { + let Some(record) = self.record else { + return; + }; + let mut state = self.pool.state.lock(); + if let Some(index) = state.position(self.slot, record) { + state.footprints[index].fragment = Some(fragment); + } + } +} + +impl Pool { + /// Under the device mutex, which orders every publication: the newest + /// published fragment kick of each other pair whose live same-VM render + /// overlaps `footprint`. Quarantined pairs are skipped as in admission. + pub(crate) fn publication_barriers( + &self, + owner: u64, + slot: u8, + footprint: &Footprint, + barriers: &mut [KickDependency; hazard::PUBLICATION_BARRIERS], + ) -> Result { + let state = self.state.lock(); + let mut slots = [0u8; hazard::PUBLICATION_BARRIERS]; + let mut count = 0; + for entry in state.footprints.iter() { + let (Some(other), Some(fragment)) = (entry.slot, entry.fragment) else { + continue; + }; + if other == slot + || entry.owner == owner + || state.quarantined & (1u64 << other) != 0 + || !footprint.same_vm(&entry.footprint) + || !footprint.overlaps(&entry.footprint) + { + continue; + } + match slots[..count].iter().position(|known| *known == other) { + // One fragment ring completes in kick order; same QID, so the + // word orders by timestamp. + Some(index) => { + if fragment.word() > barriers[index].word() { + barriers[index] = fragment; + } + } + // Admission bounds the pairs; see hazard::plan. + None if count == hazard::PUBLICATION_BARRIERS => return Err(EBUSY), + None => { + slots[count] = other; + barriers[count] = fragment; + count += 1; + } + } + } + Ok(count) + } +} + impl Drop for RenderLease { fn drop(&mut self) { let mut state = self.pool.state.lock(); - if let Some((owner, sequence)) = self.record { - if let Some(index) = state.footprints.iter().position(|entry| { - entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence - }) { + if let Some(record) = self.record { + if let Some(index) = state.position(self.slot, record) { state.footprints.swap_remove(index); } } state.release(self.slot); drop(state); - self.pool.released.notify_all(); + self.pool.returned(); } } @@ -756,7 +897,7 @@ impl Pool { return; } self.state.lock().kill_hold &= !slots; - self.released.notify_all(); + self.returned(); } } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 713809907dd4b..9c4c5c92d049d 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -15,7 +15,7 @@ use crate::g17::{ queue::*, variant::PointerLayout, }, - object::{Allocator, CpuMap, KernelObject}, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, }; use crate::{hw::t8140::dynamic, mmu}; use core::sync::atomic::{fence, Ordering}; @@ -30,10 +30,20 @@ const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; +const NATIVE_WAIT_OFFSET: usize = 0x900; +const NATIVE_WAIT_STRIDE: usize = 0x60; +// A submission can name every physical compute channel. Keep the complete +// wait list inside its own cached slot, including the last retained slot. +const CACHED_RECORD_STRIDE: usize = (NATIVE_WAIT_OFFSET + + crate::g17::kick::COMPUTE_CHANNELS * NATIVE_WAIT_STRIDE).next_power_of_two(); const CONFIG: usize = 0; const ANNOUNCE: usize = 0x800; const PREDECESSOR: usize = 0x400; const POINTERS: usize = 0x100; +/// Producer/consumer words live on their own uncached page after the graph, +/// which the firmware reads through a cached mapping (queue record, job-list +/// head and item ring), as native does. +const CURSORS: usize = GRAPH_SIZE; const ITEMS: usize = 0x4000; const STAMP: usize = 0x300; @@ -83,24 +93,32 @@ pub(crate) struct Binding { /// Descriptor ring of one compute QID with its kernel-VM alias. Built without /// the device mutex; installed into a vacant graph before an owner binds. pub(crate) struct DescriptorRing { - object: KernelObject, - high: mmu::KernelMapping, + // Drop the GPU alias before the retained storage returns to its pool. + low: mmu::KernelMapping, + object: PooledObject, } impl DescriptorRing { - pub(crate) fn new(alloc: &Allocator<'_>) -> Result { - let page = mmu::UAT_PGSZ as u64; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let object = alloc.lower( - DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - page, - prot, + /// Firmware reads every descriptor through a cached kernel mapping, as it + /// does for render. The pool keeps that mapping until processor stop; the + /// hardware reads the same pages through an uncached canonical low alias. + pub(crate) fn new(alloc: &Allocator<'_>, objects: &Arc) -> Result { + let size = DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE; + let object = objects.allocate( + alloc, + Family::Descriptor, + size, + size, + mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; - Ok(Self { object, high }) + let low = object.alias_in( + alloc.uat.kernel_lower_vm(), + dynamic::LOWER, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + )?; + Ok(Self { low, object }) } } @@ -112,10 +130,14 @@ struct Graph { usage: KernelObject, operand: KernelObject, support: KernelObject, - graph: KernelObject, + graph: PooledObject, context_low: KernelObject, context_high: KernelObject, + /// Announcements, which stay uncached for the firmware. records: KernelObject, + /// Configuration, predecessor and completion-wait records at the same + /// offsets, which the firmware reads through a cached mapping. + cached_records: PooledObject, record_ends: [Option; RECORD_SLOTS], reclaimed: u32, /// Host words of the pointer block in this firmware build. @@ -127,6 +149,7 @@ struct Graph { impl Graph { fn new( alloc: &Allocator<'_>, + objects: &Arc, context: &Context, pointers: PointerLayout, operand_state: &'static OperandState, @@ -134,7 +157,7 @@ impl Graph { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let ring = DescriptorRing::new(alloc)?; + let ring = DescriptorRing::new(alloc, objects)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(operand_state.size, page, prot, cpu)?; @@ -144,11 +167,18 @@ impl Graph { support.write(0x100, support_va + page + 4)?; support.write(0x110, 0x50u32)?; support.write(mmu::UAT_PGSZ + 4, 1u32)?; - let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; + let mut graph = objects.allocate( + alloc, + Family::QueueState, + GRAPH_SIZE + mmu::UAT_PGSZ, + GRAPH_SIZE, + prot, + cpu, + )?; let graph_va = graph.gpu_va(); graph.initialize::(0, |view| { view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + POINTERS as u64, + pointers_va: graph_va + CURSORS as u64, items_va: graph_va + ITEMS as u64, job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), @@ -159,9 +189,21 @@ impl Graph { view.jobs = JobListHead::new(graph_va + 0x200); Ok(()) })?; + graph.initialize::(CURSORS, |view| { + *view = PointerBlock::new(&pointers)?; + Ok(()) + })?; let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + let cached_records = objects.allocate( + alloc, + Family::QueueConfig, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + RECORD_SLOTS * CACHED_RECORD_STRIDE, + prot, + cpu, + )?; Ok(Self { ring: Some(ring), preempt, @@ -172,6 +214,7 @@ impl Graph { context_low, context_high, records, + cached_records, record_ends: [None; RECORD_SLOTS], reclaimed: 0, pointers, @@ -237,9 +280,9 @@ impl Graph { fn cursors(&self) -> Result { Ok(Cursors { - producer: self.graph.word(POINTERS + self.pointers.producer)?.load(Ordering::Relaxed), - consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), - count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), + producer: self.graph.word(CURSORS + self.pointers.producer)?.load(Ordering::Relaxed), + consumer: self.graph.word(CURSORS)?.load(Ordering::Relaxed), + count: self.graph.word(CURSORS + 0x60)?.load(Ordering::Relaxed), }) } fn stamp(&self) -> Result { @@ -250,12 +293,12 @@ impl Graph { fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; - Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) + Ok((ring.object.gpu_va() + offset, ring.low.iova() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { let ring = self.ring.as_ref().ok_or(EAGAIN)?; ring.object - .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) + .map_alias(vm, ring.low.iova(), mmu::PROT_GPU_SHARED_RO) } /// Detaches the descriptor ring of an idle vacant queue; the caller frees it /// off-lock. The job list and the context item only name consumed @@ -293,7 +336,7 @@ impl Graph { } fence(Ordering::SeqCst); self.graph - .word(POINTERS + self.pointers.producer)? + .word(CURSORS + self.pointers.producer)? .store(cursors.consumer, Ordering::Relaxed); self.reclaimed = cursors.consumer; fence(Ordering::SeqCst); @@ -326,6 +369,10 @@ use crate::g17::{ }; use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; +/// Position in its queue's active ring from which a publication withholds its doorbell: +/// with two unretired commands ahead, it cannot be the next to start. +const WITHHOLD_DEPTH: usize = 3; + struct Fifo { entries: [Option; IN_FLIGHT], head: usize, @@ -523,8 +570,14 @@ pub(crate) struct Queue { /// Bit `p` set once a work slot of priority class `p` named this queue. outer_started: u8, published_config: Option, + /// Last configuration record the firmware consumed, and its recovery + /// epoch. A command repeats the record only when it differs. + config_cache: Option<(QueueConfig, u64)>, ordinal: u32, submitted: u32, + /// Prepared packets of the bound owner about to publish (see [`Self::claim`]). + claims: u32, + last_published: u64, last_end: u64, released: bool, retirement_ready: bool, @@ -538,13 +591,67 @@ pub(crate) struct Queue { quarantine_error: Option, failure_status_pending: bool, retirement_proved: bool, + source_readers: Arc, + source_hint: crate::g17::sync_input::ReaderHint, + /// The doorbell withheld from publications that had [`WITHHOLD_DEPTH`] - 1 or more + /// unretired commands of this queue ahead of them, the publication of the first, and the + /// compute work slots written at its priority once its slot was. + withheld_doorbell: Option<(u64, u64, u64)>, } impl Queue { + /// Returns the withheld doorbell once at most one command of this queue that precedes the + /// first publication it covers has no completion record yet. A doorbell of the same class + /// rung after the last withheld work slot was written announced every one of them: nothing + /// is left to ring. `announced` gives the slots announced per class. + /// + /// Retirement waiting for a stamp does not hold the doorbell back, and any departure from + /// ordinary retirement (timeout, kill, quarantine, replay, recovery or exit) rings it at + /// once: the firmware then sees every published command, as without withholding. + pub(crate) fn due_doorbell(&mut self, announced: impl Fn(u8) -> u64) -> Option { + let (doorbell, first, written) = self.withheld_doorbell?; + if announced(((doorbell >> 2) & 3) as u8) >= written { + self.withheld_doorbell = None; + return None; + } + let settling = self.settling(); + let incomplete = self + .active + .iter() + .filter(|active| active.published < first && active.observed.is_none()) + .count(); + if !settling && incomplete > 1 { + return None; + } + self.withheld_doorbell = None; + Some(doorbell) + } + + /// The queue has left ordinary retirement: a timeout, kill, quarantine, replay, recovery + /// witness or exit is in progress. + fn settling(&self) -> bool { + self.quarantined + || self.released + || self.exit.is_some() + || self.replays.len != 0 + || self.replay_retirement + || self.awaiting_witness + || self.retire_pending + || self.retired_by_teardown + || self.spared_quarantine + || self.failure_status_pending + || self.terminal.is_some() + } + + /// Takes the withheld doorbell regardless of its predecessors. + pub(crate) fn take_withheld_doorbell(&mut self) -> Option { + self.withheld_doorbell.take().map(|(doorbell, _, _)| doorbell) + } /// The caller reserves a QID first, cancels it on construction failure, and retains every /// successfully installed queue until processor stop. Shared VM apertures precede this call. pub(crate) fn new( alloc: &Allocator<'_>, + objects: &Arc, backing: Backing, owner: u64, context: Arc, @@ -552,11 +659,13 @@ impl Queue { operand_state: &'static OperandState, ) -> Result { let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context, pointers, operand_state)?; + let mut graph = Graph::new(alloc, objects, &context, pointers, operand_state)?; let kick_alias = kick.map_client(context.vm())?; let pool_alias = pool.map_client(context.vm())?; let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; context.mark_published(); + context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, kick.id().qid(), context.id(), 0); Ok(Self { owner: Some(owner), retained_owner: None, @@ -582,8 +691,11 @@ impl Queue { installed: false, outer_started: 0, published_config: None, + config_cache: None, ordinal: 0, submitted: 0, + claims: 0, + last_published: 0, last_end: 0, released: false, retirement_ready: false, @@ -597,6 +709,9 @@ impl Queue { quarantine_error: None, failure_status_pending: false, retirement_proved: false, + source_readers: Arc::new(core::sync::atomic::AtomicU32::new(0), GFP_KERNEL)?, + source_hint: crate::g17::sync_input::ReaderHint::new(), + withheld_doorbell: None, }) } pub(crate) fn qid(&self) -> u8 { @@ -659,6 +774,27 @@ impl Queue { visit(active.packet.completion.status()); } } + /// A recovery record names this queue and the completion stamp of the + /// command it interrupted, `kick << 8` like the queue's stamp. Without a + /// stamp match the oldest published command is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self.active.iter().find(|active| { + active + .ticket + .is_some_and(|ticket| (ticket.kick as u32) & 0xff_ffff == sequence) + }) { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.ticket.is_some()) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { for active in self.active.iter() { active @@ -706,33 +842,32 @@ impl Queue { || self.retire_pending || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) } - /// Stage every unlatched exact identity before the device's single shared-ring scan. - pub(crate) fn request_observations( - &self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + /// Stage every unlatched exact identity before the device's single shared-ring scan, + /// whichever queues the scan's batch retires. + pub(crate) fn request_observations(&self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(false); } + batch.stage_frontier(self.qid(), self.kick.timestamp().get())?; let mut requested = false; + let mut stamp = None; for active in self.active.iter() { let Some(ticket) = active.ticket else { break; }; if active.observed.is_none() { - batch.request(&ticket)?; + let stamp = match stamp { + Some(stamp) => stamp, + None => *stamp.insert(self.graph.stamp()?), + }; + batch.request(&ticket, retirement::stamp_covers(stamp, ticket.kick))?; requested = true; } } Ok(requested) } - pub(crate) fn latch_observations( - &mut self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { + pub(crate) fn latch_observations(&mut self, batch: &mut Batch) -> Result { + if self.quarantined { return Ok(()); } for active in self.active.iter_mut() { @@ -887,6 +1022,11 @@ impl Queue { self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; + if let Some(ticket) = front.ticket { + crate::g17::recovery::note_retired(); + crate::g17::trace::complete(crate::g17::trace::COMPUTE, ticket.qid, + ticket.ordinal, &front.packet, [timestamps[0], timestamps[1]]); + } self.retire_pending = true; defer(Deferred::Retired( front.packet.completion.clone(), @@ -1139,6 +1279,32 @@ fn release_pending(released: bool, quarantined: bool, active: usize, retiring: b mod admission_tests { use super::*; + #[test] + fn complete_native_wait_list_fits_every_cached_slot() { + let waits = crate::g17::kick::COMPUTE_CHANNELS; + for slot in 0..RECORD_SLOTS { + for index in 0..waits { + let start = record_start(slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).unwrap(); + assert!(start >= slot * CACHED_RECORD_STRIDE); + assert!(start + NATIVE_WAIT_STRIDE <= (slot + 1) * CACHED_RECORD_STRIDE); + } + } + } + + #[test] + fn record_window_cannot_overwrite_the_next_slot() { + // The nineteenth wait used to cross the old slot boundary silently. + let offset = NATIVE_WAIT_OFFSET + 18 * NATIVE_WAIT_STRIDE; + assert_eq!(record_start(0, offset, NATIVE_WAIT_STRIDE, RECORD_STRIDE), Err(EINVAL)); + assert!(record_start(0, offset, NATIVE_WAIT_STRIDE, CACHED_RECORD_STRIDE).is_ok()); + assert_eq!(record_start(RECORD_SLOTS, 0, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, usize::MAX, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, CACHED_RECORD_STRIDE, 1, CACHED_RECORD_STRIDE), Err(EINVAL)); + assert_eq!(record_start(0, CACHED_RECORD_STRIDE - 1, 1, CACHED_RECORD_STRIDE), + Ok(CACHED_RECORD_STRIDE - 1)); + } + #[test] fn retained_ring_pressure_waits_without_discarding_work() { let mut active = Fifo::new(); @@ -1230,11 +1396,15 @@ impl Queue { && self.active.len == 0 && self.replays.len == 0 && self.previous.is_none() + // Borrow acquisition is serialized by the device mutex. Avoid + // touching the separate loan counter for an otherwise busy queue. + && !self.source_hint.has_readers(&self.source_readers) } pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { if !self.reusable() { return Err(EBUSY); } + self.source_hint.reset_for_owner(&self.source_readers)?; if !context.is_current() { return Err(EFAULT); } @@ -1274,6 +1444,9 @@ impl Queue { self.retained_owner = None; } context.mark_published(); + context.note_compute_binding(true); + crate::g17::trace::bind(crate::g17::trace::COMPUTE, self.qid(), context.id(), + if same_vm { 2 } else { 1 }); self.owner = Some(owner); self.unpublished_binding = true; self.released = false; @@ -1284,6 +1457,9 @@ impl Queue { /// True allows the registry to cancel the never-published QID and drop this graph. An /// installed graph remains retained even after the logical owner exits without work. pub(crate) fn release_owner(&mut self) -> Result { + if let Some(binding) = &self.binding { + binding.context.note_compute_binding(false); + } if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { return Ok(true); } @@ -1338,7 +1514,9 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() + self.pool_phase == PoolPhase::Populated && self.pool.populated() + && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } /// A recycled killed queue still holding the backing its killed work used. pub(crate) fn recycled_pending(&self) -> bool { @@ -1396,6 +1574,7 @@ impl Queue { } fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; + self.config_cache = None; self.exit = None; self.quarantined = false; self.quarantine_error = None; @@ -1476,6 +1655,7 @@ impl Queue { ring: DescriptorRing, ) -> Result<(u16, u64)> { if !self.can_rearm() { return Err(EAGAIN); } + self.withheld_doorbell = None; self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; @@ -1634,11 +1814,11 @@ impl Queue { let producer = self .graph .graph - .word(POINTERS + self.graph.pointers.producer)? + .word(CURSORS + self.graph.pointers.producer)? .load(Ordering::Relaxed); self.graph .graph - .word(POINTERS + target)? + .word(CURSORS + target)? .store(producer, Ordering::Relaxed); Ok(()) } @@ -1690,6 +1870,15 @@ impl Queue { } } +/// Validate the individual slot before deriving an offset into its allocation. +/// Allocation-wide checks cannot detect a write into the following live slot. +fn record_start(slot: usize, offset: usize, window: usize, stride: usize) -> Result { + if slot >= RECORD_SLOTS || offset > stride || window > stride - offset { + return Err(EINVAL); + } + slot.checked_mul(stride).and_then(|start| start.checked_add(offset)).ok_or(EINVAL) +} + impl Graph { fn record( &mut self, @@ -1699,12 +1888,15 @@ impl Graph { value: T, ) -> Result { let (object, start) = if let Some(slot) = slot { - if slot >= RECORD_SLOTS { - return Err(EINVAL); - } - (&mut self.records, slot * RECORD_STRIDE + offset) + let (object, stride): (&mut KernelObject, usize) = if offset == ANNOUNCE { + (&mut self.records, RECORD_STRIDE) + } else { + (&mut self.cached_records, CACHED_RECORD_STRIDE) + }; + (object, record_start(slot, offset, window, stride)?) } else { - (&mut self.graph, 0x6800 + offset) + let start = record_start(0, offset, window, GRAPH_SIZE - 0x6800)?; + (&mut *self.graph, 0x6800 + start) }; if core::mem::size_of::() > window { return Err(EINVAL); @@ -1730,7 +1922,7 @@ impl Graph { return Err(EBUSY); } let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; if slot.load(Ordering::Relaxed) != 0 { return Err(EBUSY); } @@ -1758,7 +1950,8 @@ pub(crate) trait Host { fn publish_qid(&mut self, id: kick::Id) -> Result; fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) -> Result; - fn notify(&mut self, message: u64) -> Result; + fn notify_compute(&mut self, priority: u8, message: u64) -> Result; + fn compute_written(&self, priority: u8) -> u64; fn note_submission(&mut self) -> Result; } @@ -1775,6 +1968,17 @@ impl Queue { dependencies: &[crate::g17::fw::kick::KickDependency], host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.publish_with_native_waits(packet, dependencies, &[], host, defer) + } + + pub(in crate::g17) fn publish_with_native_waits( + &mut self, + packet: Arc, + dependencies: &[crate::g17::fw::kick::KickDependency], + native: &[crate::g17::sync_input::NativeWait], + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { use crate::g17::fw::{ channels::WorkSlot, @@ -1809,11 +2013,22 @@ impl Queue { let (descriptor_high, descriptor_low) = self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; + // The first prototype only waits on an installed consumer channel. + // A retry without a published source falls back to completion on CPU. + if first && !native.is_empty() { return Err(EAGAIN); } + if native.len() > crate::g17::kick::COMPUTE_CHANNELS { return Err(EINVAL); } let entry_signal = first || dependencies .iter() .any(|dependency| dependency.qid() != qid); item_capacity(self.graph.cursors()?, entry_signal)?; + if !native.is_empty() { + let cursors = self.graph.cursors()?; + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < (if entry_signal { 4 } else { 3 }) + native.len() as u32 { + return Err(EAGAIN); + } + } let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); if let Some(slot) = record_slot { self.graph.ensure_record_slot(slot)?; @@ -1858,6 +2073,8 @@ impl Queue { let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; + let cached = self.config_cache.and_then(|(config, at)| (at == epoch).then_some(config)); + let send_config = first || super::render::publication::required(cached, config); let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -1879,6 +2096,7 @@ impl Queue { priority: policy.priority(), }; let published = host.next_compute_publication()?; + self.last_published = published; self.active .push(Active { packet: packet.clone(), @@ -1904,7 +2122,7 @@ impl Queue { .object .initialize::( kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), + |descriptor| descriptor.write_zeroed(&args), )?; if first { self.graph.context_high.write( @@ -1921,8 +2139,20 @@ impl Queue { let cursors = self.graph.cursors()?; let _ = self.graph.recycle(cursors); } - let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; + let config_va = if send_config { + Some(self.graph.record(record_slot, CONFIG, 0xc0, config)?) + } else { + None + }; let cursors = self.graph.cursors()?; + let mut wait_vas = KVec::with_capacity(native.len(), GFP_KERNEL)?; + for (index, wait) in native.iter().enumerate() { + let record = wait.bytes(crate::g17::fw::stamp_value(kick_timestamp.get() as u32), + packet.order.sequence); + let address = self.graph.record(record_slot, NATIVE_WAIT_OFFSET + index * NATIVE_WAIT_STRIDE, + NATIVE_WAIT_STRIDE, record)?; + wait_vas.push(address, GFP_KERNEL)?; + } let prepared_kick = self.kick.prepare_entry(&kick_args)?; let mut producer; if first { @@ -1948,7 +2178,7 @@ impl Queue { })?; producer = self .graph - .append(producer, config_va, || {}) + .append(producer, config_va.ok_or(EINVAL)?, || {}) .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; @@ -1957,15 +2187,22 @@ impl Queue { })?; } else { let target = self.pool.reserve_submission()?; - producer = self - .graph - .append(cursors.producer, descriptor_high, || { - packet.mark_published() + producer = (|| -> Result { + let mut before_descriptor = cursors.producer; + for &wait in &wait_vas { + before_descriptor = self.graph.append(before_descriptor, wait, || { + packet.mark_published(); + visible = true; + })?; + } + self.graph.append(before_descriptor, descriptor_high, || { + packet.mark_published(); + visible = true; }) - .or_else(|error| { - self.pool.cancel_unpublished(target)?; - Err(error) - })?; + })().or_else(|error| { + if !visible { self.pool.cancel_unpublished(target)?; } + Err(error) + })?; visible = true; self.unpublished_binding = false; let qos = host.qos_publish( @@ -1978,14 +2215,16 @@ impl Queue { policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; + if let Some(config_va) = config_va { + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; + Err(error) + })?; + } } if first { prepared_kick.prepare_compute_install()?; @@ -2015,10 +2254,12 @@ impl Queue { self.outer_started & ring == 0, )?, )?; - let notify_activation = self.outer_started & ring == 0; self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); + if send_config { + self.config_cache = Some((config, epoch)); + } self.ordinal = ordinal; let (accepted, state) = host.epoch()?; if state != 0 || accepted != epoch { @@ -2043,13 +2284,34 @@ impl Queue { previous.publication = Some((packet.order.sequence, epoch)); } } - if notify_activation { - fence(Ordering::SeqCst); - // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; - } + // A publication rings its own pipe and priority class, as render + // publication does: pipe 2 (compute) in bits 1:0, priority class in + // bits 3:2. The generic work doorbell left the firmware to find the + // ring itself and delayed dispatch. + // + // A doorbell makes the firmware read the whole work ring of its class, + // and each one it handles while finishing a command delays that + // command's completion and the release of its waiters. A command with + // two or more unretired commands of its queue ahead cannot start soon: + // its doorbell is withheld until at most one precedes it (see + // `due_doorbell`), and any doorbell rung meanwhile on the same ring + // announces its work slot earlier. fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x10)?; + let doorbell = (0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2; + if self.active.len >= WITHHOLD_DEPTH + && !self.settling() + && self.withheld_doorbell.is_none_or(|(withheld, _, _)| withheld == doorbell) + { + let written = host.compute_written(policy.priority()); + let entry = self.withheld_doorbell.get_or_insert((doorbell, published, written)); + entry.2 = written; + } else { + host.notify_compute(policy.priority(), doorbell)?; + if self.withheld_doorbell.is_some_and(|(withheld, _, _)| withheld == doorbell) { + self.withheld_doorbell = None; + } + } + crate::g17::trace::publish(crate::g17::trace::COMPUTE, qid, ordinal, &packet); host.note_submission() })(); if let Err(error) = result { @@ -2155,7 +2417,7 @@ impl Graph { let first = self.graph.dword(ITEMS)?; let second = self.graph.dword(ITEMS + 8)?; let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + self.pointers.producer)?; + let producer = self.graph.word(CURSORS + self.pointers.producer)?; first.store(address, Ordering::Relaxed); second.store(0, Ordering::Relaxed); third.store(0, Ordering::Relaxed); @@ -2262,21 +2524,57 @@ impl Queue { /// Logical queue/context ownership is unchanged. The existing binding is /// retained through the successor's exact replacement witness. pub(crate) fn share_idle(&mut self) -> Result { - if !self.installed || self.owner.is_none() || !self.retirement_ready - || self.growing || self.quarantined || self.awaiting_witness - || self.exit.is_some() || self.active.len != 0 || self.replays.len != 0 - || self.previous.is_some() || self.pool_phase != PoolPhase::Populated - { + if !self.shareable() { return Ok(false); } self.release_owner()?; Ok(self.reusable()) } + + /// Another owner could take this channel now: idle and released, shareable, or vacant. + pub(crate) fn channel_free(&self) -> bool { + self.reusable() || self.shareable() || self.can_rearm() + } + + /// A pool replacement is in flight for this queue. + pub(crate) fn growing(&self) -> bool { + self.growing + } + + /// The queue's first command has been published: its identity and configuration exist. + pub(crate) fn installed(&self) -> bool { + self.installed + } + + /// A packet of the bound owner passed prepare_job and will publish here. + pub(crate) fn claim(&mut self) { + self.claims = self.claims.saturating_add(1); + } + + pub(crate) fn release_claim(&mut self) { + self.claims = self.claims.saturating_sub(1); + } + + /// An open client's idle channel that [`Self::share_idle`] can hand over. + pub(crate) fn shareable(&self) -> bool { + self.claims == 0 && self.installed && self.owner.is_some() && self.retirement_ready + && !self.growing && !self.quarantined && !self.awaiting_witness + && self.exit.is_none() && self.active.len == 0 && self.replays.len == 0 + && self.previous.is_none() && self.pool_phase == PoolPhase::Populated + && !self.source_hint.has_readers(&self.source_readers) + } + + /// Device publication order of this channel's newest command: lower is + /// idle longer. + pub(crate) fn last_published(&self) -> u64 { + self.last_published + } } impl Queue { pub(crate) fn can_rearm(&self) -> bool { self.pool_phase == PoolPhase::Vacant && (self.reusable() || self.recycled_pending()) + && !self.source_hint.has_readers(&self.source_readers) } } @@ -2318,3 +2616,50 @@ impl Queue { Ok((false, retired)) } } + + +impl Queue { + /// Same immutable singleton identity and current physical epoch. + pub(crate) fn has_input_source(&self, source: &Arc, + status: &Arc, epoch: u64) -> bool + { + !self.quarantined && self.replays.len == 0 && self.active.iter().any(|active| + active.ticket.is_some_and(|ticket| ticket.epoch == epoch) + && Arc::ptr_eq(active.packet.completion.status(), status) + && active.packet.completion.is_input_source(source)) + } + + pub(crate) fn source_dependency(&mut self, source: &crate::g17::sync_input::Source, + status: &Arc, epoch: u64) -> Result> + { + for active in self.active.iter() { + if !active.packet.completion.is_input_source(&source.submission) { continue; } + if !Arc::ptr_eq(active.packet.completion.status(), status) { return Err(EINVAL); } + if self.quarantined || self.replays.len != 0 { return Err(EAGAIN); } + let ticket = active.ticket.ok_or(EAGAIN)?; + if ticket.epoch != epoch { return Err(EAGAIN); } + self.valid_ticket(active, &ticket)?; + if active.packet.cancel_requested() { return Err(ECANCELED); } + source.submission.source_status()?; + let dependency = crate::g17::fw::kick::KickDependency::new(self.qid(), + crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?).ok_or(EIO)?; + let hold = self.source_hint.hold(&self.source_readers)?; + return Ok(Some((active.published, dependency, + self.graph.support.gpu_va() + STAMP as u64, + u32::from(self.qid()), hold))); + } + Ok(None) + } + + /// Preparing a possible wait does not create a firmware consumer. Mark it + /// only once the consumer's first item is visible, under the device mutex. + pub(crate) fn note_input_consumer(&self, source: &crate::g17::sync_input::Source) { + for active in self.active.iter() { + if active.packet.completion.is_input_source(&source.submission) { + active.packet.completion.note_firmware_consumer(); + break; + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index e938ec566dbac..c2b49e7b436d7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,7 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. -use crate::g17::initdata::ComputeCompletion; +use crate::g17::initdata::{CompletionFold, ComputeCompletion}; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -161,35 +161,122 @@ pub(crate) fn quarantined_retired( && submitted == pending) } +/// Walks a completion ring of `seen.len()` slots from `start` over the records written since +/// the previous walk: `seen` holds each slot's control word as of the last walk that passed it, +/// so the first slot still holding it is where the firmware writes next. `record` folds one +/// slot; the walk also ends once it reports every request found, or at a requested record +/// still being written, which the next walk reads again. Returns where the next walk starts. +pub(crate) fn walk_completions( + start: usize, + seen: &mut [u64], + mut control: impl FnMut(usize) -> Result, + mut record: impl FnMut(usize) -> Result, +) -> Result { + let slots = seen.len(); + if slots == 0 { + return Err(EINVAL); + } + let start = start % slots; + for step in 0..slots { + let slot = (start + step) % slots; + let word = control(slot)?; + if word == seen[slot] { + return Ok(slot); + } + match record(slot)? { + CompletionFold::Incomplete => return Ok(slot), + CompletionFold::Passed => seen[slot] = word, + CompletionFold::Done => { + seen[slot] = word; + return Ok((slot + 1) % slots); + } + } + } + Ok(start) +} + #[derive(Clone, Copy)] struct Request { descriptor: u64, kick: u64, observed: Option, + covered: bool, + next: u16, +} + +const NO_REQUEST: u16 = u16::MAX; +const _: () = assert!(QIDS * IN_FLIGHT < NO_REQUEST as usize); + +fn request_bucket(qid: usize, kick: u64) -> usize { + qid * IN_FLIGHT + kick as usize % IN_FLIGHT } /// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; /// staging and folding never allocate under the device mutex. pub(crate) struct Batch { requests: KVVec, + /// Per-QID kick buckets; collisions retain the complete identity check. + heads: KVVec, slots: [(usize, usize); QIDS], open: Option, + /// Requests with no observation yet; the scan stops when this reaches zero. + unobserved: usize, + /// Unobserved requests whose queue's completion stamp already covered them when staged. + /// The firmware stores the stamp after the record, so each has a record in the ring. + covered: usize, + /// Per QID, the next kick its queue had not published when the batch was staged + /// ([`NO_FRONTIER`] for a QID without a staged queue). A record at or past it belongs + /// to a command published during the scan, which a later batch stages. + frontiers: [u64; QIDS], } +const NO_FRONTIER: u64 = u64::MAX; +const KICK_MASK: u64 = (1 << 40) - 1; + impl Batch { pub(crate) fn new() -> Result { + let mut heads = KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?; + for _ in 0..QIDS * IN_FLIGHT { + heads.push_within_capacity(NO_REQUEST).map_err(|_| ENOSPC)?; + } Ok(Self { requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, + heads, slots: [(0, 0); QIDS], open: None, + unobserved: 0, + covered: 0, + frontiers: [NO_FRONTIER; QIDS], }) } pub(crate) fn clear(&mut self) { self.requests.clear(); self.slots.fill((0, 0)); self.open = None; + self.unobserved = 0; + self.covered = 0; + self.frontiers.fill(NO_FRONTIER); + } + /// Records the first kick `qid`'s queue had not yet published when this batch was staged. + pub(crate) fn stage_frontier(&mut self, qid: u8, next: u64) -> Result { + *self.frontiers.get_mut(usize::from(qid)).ok_or(EINVAL)? = next & KICK_MASK; + Ok(()) + } + /// A record of `qid` with `kick` names a command published after this batch was staged. + fn published_after_staging(&self, qid: usize, kick: u64) -> bool { + let frontier = self.frontiers[qid]; + frontier != NO_FRONTIER && kick.wrapping_sub(frontier) & KICK_MASK < 1 << 39 + } + /// True once every requested identity has an observation. + pub(crate) fn complete(&self) -> bool { + self.unobserved == 0 } - pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { + /// Some request whose record must already be in the ring is still unobserved. + pub(crate) fn covered_pending(&self) -> bool { + self.covered != 0 + } + /// Stages `ticket`; `covered` when its queue's completion stamp already covers its kick. + pub(crate) fn request(&mut self, ticket: &Ticket, covered: bool) -> Result { if ticket.descriptor == 0 || ticket.kick == 0 { return Err(EINVAL); } @@ -201,65 +288,119 @@ impl Batch { return Err(ENOSPC); } let start = if slot.1 == 0 { + let begin = ticket.qid as usize * IN_FLIGHT; + self.heads.get_mut(begin..begin + IN_FLIGHT).ok_or(EIO)?.fill(NO_REQUEST); self.requests.len() } else { slot.0 }; + let bucket = request_bucket(ticket.qid as usize, ticket.kick); + let next = *self.heads.get(bucket).ok_or(EIO)?; + let index = u16::try_from(self.requests.len()).map_err(|_| ENOSPC)?; self.requests .push_within_capacity(Request { descriptor: ticket.descriptor, kick: ticket.kick, observed: None, + covered, + next, }) .map_err(|_| ENOSPC)?; + *self.heads.get_mut(bucket).ok_or(EIO)? = index; *slot = (start, slot.1 + 1); self.open = Some(ticket.qid); + self.unobserved += 1; + self.covered += usize::from(covered); Ok(()) } - /// Capture the control word once, then read only a requested identity's payload. - /// Records can be overwritten during a scan: incomplete or older identities are skipped. - pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - let control = record.read(0)?; + /// Capture the control word once. Most slots belong to an older kick of a + /// selected QID; reject those before reading any additional shared words. + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + if !self.fold_reader(|word| record.read(word))? { + return Ok(CompletionFold::Incomplete); + } + Ok(self.folded()) + } + + /// Like [`Self::fold`], for a scan that reads every slot regardless of the outcome. + pub(crate) fn fold_any(&mut self, record: ComputeCompletion<'_>) -> Result { + self.fold_reader(|word| record.read(word)).map(|_| ()) + } + + fn folded(&self) -> CompletionFold { + if self.complete() { CompletionFold::Done } else { CompletionFold::Passed } + } + + /// Ok(false) when a pending request's record is still being written. + fn fold_reader(&mut self, mut read: impl FnMut(usize) -> Result) -> Result { + let control = read(0)?; let qid = ((control >> 9) & 0x7f) as usize; - let (start, count) = self.slots[qid]; + let (_, count) = self.slots[qid]; + let kick = + (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; if count == 0 { - return Ok(()); + return Ok(!self.published_after_staging(qid, kick)); } - let descriptor = record.read(6)? & !1; - if descriptor == 0 || record.read(7)? == 0 { - return Ok(()); + let mut index = *self.heads.get(request_bucket(qid, kick)).ok_or(EIO)?; + let mut descriptor = None; + let mut found = None; + for _ in 0..count { + if index == NO_REQUEST { + break; + } + let request = self.requests.get(usize::from(index)).ok_or(EIO)?; + if request.kick == kick { + let value = if let Some(value) = descriptor { + value + } else { + let value = read(6)? & !1; + if value == 0 || read(7)? == 0 { + return Ok(request.observed.is_some()); + } + descriptor = Some(value); + value + }; + if request.descriptor == value { + // Chains run newest first. Retain the original first-match + // rule even if a caller stages the same identity twice. + found = Some(usize::from(index)); + } + } + index = request.next; } - let kick = - (((control >> 8) & 0x00ff_ffff_ff00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; - let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; - let Some(request) = requests - .iter_mut() - .find(|r| r.descriptor == descriptor && r.kick == kick) - else { - return Ok(()); + let Some(index) = found else { + return Ok(!self.published_after_staging(qid, kick)); }; - let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; - if timestamps[0] == 0 - || timestamps[1] == 0 - || request - .observed - .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + let request = self.requests.get_mut(index).ok_or(EIO)?; + let timestamps = [read(2)? & TIME_MASK, read(5)? & TIME_MASK]; + if timestamps[0] == 0 || timestamps[1] == 0 { + return Ok(request.observed.is_some()); + } + if request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) { - return Ok(()); + return Ok(true); + } + if request.observed.is_none() { + self.unobserved -= 1; + self.covered -= usize::from(request.covered); } request.observed = Some(Observation { timestamps }); - Ok(()) + Ok(true) } + + /// A command published after this batch staged its queue's requests was not part of the + /// ring snapshot: it stays unobserved until a later batch. pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; - let request = self + Ok(self .requests .get_mut(start..start + count) .ok_or(EIO)? .iter_mut() .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) - .ok_or(EIO)?; - Ok(request.observed.take()) + .and_then(|request| request.observed.take())) } } @@ -402,3 +543,156 @@ mod tests { assert_eq!(result, Err(ENODATA)); } } + + +#[kunit_tests(asahi_g17_completion_index)] +mod index_tests { + use super::*; + + fn ticket(qid: u8, kick: u64, descriptor: u64) -> Ticket { + Ticket { owner: 1, submission: 1, context_id: 1, qid, descriptor, + work_node: 0x2000, kick, ordinal: 1, item_producer: 4, item_count: 64, epoch: 1 } + } + fn record(ticket: &Ticket, timestamps: [u64; 2]) -> [u64; 8] { + let mut words = [0; 8]; + words[0] = ((ticket.kick & !0xff) << 8) | ((ticket.kick & 0xff) << 48) + | (u64::from(ticket.qid) << 9); + words[2] = timestamps[0]; words[5] = timestamps[1]; + words[6] = ticket.descriptor | 1; words[7] = 1; + words + } + #[test] + fn stale_kick_reads_only_the_control_even_on_a_bucket_collision() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); + let words = record(&ticket(7, 17, 0x1000), [10, 20]); + batch.fold_reader(|word| { + assert_eq!(word, 0); + Ok(words[word]) + }).unwrap(); + assert!(!batch.complete()); + } + #[test] + fn collisions_and_reused_descriptors_keep_full_kick_identity() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 49, 0x1000); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); + let words = record(&first, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(!batch.complete()); + let words = record(&second, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.take(&second).unwrap().unwrap().timestamps, [30, 40]); + } + #[test] + fn same_kick_still_checks_descriptor_and_zero_duration_is_retained() { + let mut batch = Batch::new().unwrap(); + let first = ticket(7, 33, 0x1000); + let second = ticket(7, 33, 0x2000); + batch.request(&first, false).unwrap(); batch.request(&second, false).unwrap(); + let words = record(&first, [20, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(&first).unwrap().unwrap().timestamps, [20, 20]); + assert!(batch.take(&second).unwrap().is_none()); + } + #[test] + fn duplicate_identity_keeps_the_first_staged_request() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, false).unwrap(); batch.request(¤t, false).unwrap(); + let words = record(¤t, [10, 20]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert_eq!(batch.take(¤t).unwrap().unwrap().timestamps, [10, 20]); + assert_eq!(batch.unobserved, 1); + } + #[test] + fn clear_restarts_buckets_and_later_publication_stays_unobserved() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 33, 0x1000), false).unwrap(); batch.clear(); + let current = ticket(7, 49, 0x1000); + batch.request(¤t, false).unwrap(); + let stale = record(&ticket(7, 33, 0x1000), [10, 20]); + batch.fold_reader(|word| Ok(stale[word])).unwrap(); + assert!(!batch.complete()); + assert!(batch.take(&ticket(7, 50, 0x2000)).unwrap().is_none()); + let words = record(¤t, [30, 40]); + batch.fold_reader(|word| Ok(words[word])).unwrap(); + assert!(batch.complete()); + } + #[test] + fn a_requested_record_still_being_written_is_read_again() { + let mut batch = Batch::new().unwrap(); + let current = ticket(7, 33, 0x1000); + batch.request(¤t, true).unwrap(); + let mut words = record(¤t, [10, 20]); + words[7] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + words[7] = 1; + words[5] = 0; + assert!(!batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.covered_pending()); + let words = record(¤t, [10, 20]); + assert!(batch.fold_reader(|word| Ok(words[word])).unwrap()); + assert!(batch.complete() && !batch.covered_pending()); + } + #[test] + fn a_command_published_during_the_scan_is_left_for_the_next_batch() { + let mut batch = Batch::new().unwrap(); + batch.stage_frontier(7, 50).unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + batch.stage_frontier(9, 3).unwrap(); + let late = record(&ticket(7, 50, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(late[word])).unwrap()); + let idle = record(&ticket(9, 3, 0x3000), [30, 40]); + assert!(!batch.fold_reader(|word| Ok(idle[word])).unwrap()); + let earlier = record(&ticket(9, 2, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(earlier[word])).unwrap()); + let wrapped = record(&ticket(7, 0xff_ffff_fff0, 0x3000), [30, 40]); + assert!(batch.fold_reader(|word| Ok(wrapped[word])).unwrap()); + } + fn walk(start: usize, seen: &mut [u64], ring: &[u64], outcome: impl Fn(usize) -> CompletionFold) + -> usize + { + walk_completions(start, seen, |slot| Ok(ring[slot]), |slot| Ok(outcome(slot))).unwrap() + } + #[test] + fn a_walk_reads_only_the_records_written_since_the_previous_one() { + let mut seen = [0u64; 8]; + let mut ring = [0u64; 8]; + assert_eq!(walk(0, &mut seen, &ring, |_| CompletionFold::Passed), 0); + ring[0] = 10; ring[1] = 11; ring[2] = 12; + let read = core::cell::Cell::new(0); + assert_eq!(walk(0, &mut seen, &ring, |_| { read.set(read.get() + 1); CompletionFold::Passed }), 3); + assert_eq!(read.get(), 3); + ring[3] = 13; + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + // A full lap later the firmware overwrote slots 4..8 and 0..2 with newer records. + for (slot, word) in [(4, 14), (5, 15), (6, 16), (7, 17), (0, 18), (1, 19)] { ring[slot] = word; } + assert_eq!(walk(4, &mut seen, &ring, |_| CompletionFold::Passed), 2); + } + #[test] + fn a_walk_stops_at_a_partial_record_and_after_the_last_request() { + let mut seen = [0u64; 8]; + let ring = [10, 11, 12, 13, 0, 0, 0, 0]; + assert_eq!(walk(0, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Incomplete } + else { CompletionFold::Passed }), 2); + assert_eq!(seen[..3], [10, 11, 0]); + assert_eq!(walk(2, &mut seen, &ring, |slot| if slot == 2 { CompletionFold::Done } + else { CompletionFold::Passed }), 3); + assert_eq!(walk(3, &mut seen, &ring, |_| CompletionFold::Passed), 4); + } + #[test] + fn an_unrequested_or_stale_record_is_passed() { + let mut batch = Batch::new().unwrap(); + batch.request(&ticket(7, 49, 0x1000), false).unwrap(); + let mut stale = record(&ticket(7, 33, 0x1000), [10, 20]); + stale[7] = 0; + assert!(batch.fold_reader(|word| Ok(stale[word])).unwrap()); + let other = record(&ticket(9, 33, 0x1000), [10, 20]); + assert!(batch.fold_reader(|word| Ok(other[word])).unwrap()); + assert!(!batch.complete() && !batch.covered_pending()); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 6a9b15789339f..16891524e25a0 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -10,7 +10,7 @@ use super::{ command::{Parser, UscWindow, Validated}, completion::{Completion, Destinations, PreparedPublication}, context::Context, - fence::{Outputs, Submission}, + fence::Outputs, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, validation::Proof, }; @@ -27,7 +27,6 @@ use kernel::{ xarray, }; -const EARLY_DEPTH: usize = 2; static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); pub(crate) mod admission; @@ -96,6 +95,9 @@ pub(crate) struct Queue { proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], + /// Finished frontier entries pruned during this ioctl. Their last references + /// can tear a whole completion down; that waits until the ioctl has published. + finished: KVec, failures: [Option<(u64, Fence)>; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. @@ -134,6 +136,7 @@ impl Queue { proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), + finished: KVec::new(), failures: [None, None], _slot: slot, }) @@ -155,11 +158,20 @@ impl Queue { fn prune_frontier(&mut self, index: usize) { let failure = &mut self.failures[index]; + // Without room the entries are released here, as they always could be. + let mut finished = if self.finished.reserve(self.frontiers[index].len(), GFP_KERNEL).is_ok() { + Some(&mut self.finished) + } else { + None + }; self.frontiers[index].retain(|(sequence, fences)| { - let status = job::fence_status(&fences.completed); + let status = fences.status(); if status == 0 { return true; } + if let Some(finished) = finished.as_mut() { + let _ = finished.push_within_capacity(fences.clone()); + } // One earliest failed ordinal covers every later prefix without // retaining its packet, mappings, or every failed fence. if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { @@ -179,7 +191,7 @@ impl Queue { command: Validated, order: Order, guard: mmu::VmJobGuard, - aggregate: &Arc, + aggregate: &super::fence::Aggregate, inputs: &[file::SyncItem], destinations: Destinations, ) -> Result { @@ -198,6 +210,22 @@ impl Queue { let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; self.ensure_compute(scratch.pages())?; } + // Only convert inputs when no independent host-prefix wait could keep + // the retained source bookkeeping queued behind an external signal. + let sources = if engine == Engine::Compute && !inputs.is_empty() + && !destinations.has_outputs() + && self.frontiers[0].iter().all(|(sequence, fences)| + !Order::contains(order.wait_through[0], *sequence) + || fences.status() > 0) + && self.frontiers[1].iter().all(|(sequence, fences)| + !Order::contains(order.wait_through[1], *sequence) + || fences.ready.raw() != fences.completed.raw() + || fences.status() > 0) + && self.failures.iter().enumerate().all(|(index, failure)| + failure.as_ref().is_none_or(|(sequence, _)| + !Order::contains(order.wait_through[index], *sequence))) + { self.backend.input_sources(inputs) } else { None }; + // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. let host_predecessors = self.frontiers[index] @@ -216,7 +244,8 @@ impl Queue { destinations, engine == Engine::Compute, host_predecessors.clone().next().is_some(), - if engine == Engine::Render { self.backend.feed() } else { None }, + self.backend.feed(), + sources.clone(), )?; let publication = completion.publication() .map(|target| PreparedPublication::new(target, host_predecessors)) @@ -277,7 +306,7 @@ impl Queue { // Successful completion includes host timestamp publication. Its // ready fence adds no ordering and would prevent shallow inline // publication. Failed and pending producers still need checking. - if job::fence_status(&fences.completed) > 0 { + if fences.status() > 0 { continue; } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; @@ -294,13 +323,23 @@ impl Queue { if job::fence_status(fence) > 0 { continue; } + if sources.as_ref().is_some_and(|sources| sources.contains_input(fence)) { + continue; + } dependencies.push(fence.clone(), GFP_KERNEL)?; checked_inputs.push(fence.clone(), GFP_KERNEL)?; } } - let early = engine == Engine::Render - && dependencies.is_empty() - && self.frontiers[index].len() < EARLY_DEPTH; + if let Some(sources) = &sources { + for ready in sources.publication_waits() { + dependencies.push(ready.clone(), GFP_KERNEL)?; + checked_inputs.push(ready.clone(), GFP_KERNEL)?; + } + } + // Publish from the ioctl whenever nothing has to wait, at any depth. + // The backend falls back to the scheduler behind any backlog or on any + // capacity miss, so submission order is preserved. + let early = dependencies.is_empty(); let packet = Packet::new( id, self.context.clone(), @@ -410,11 +449,17 @@ impl crate::queue::Queue for Queue { return Err(EIO); } let (mut pending, guards) = self.prepare_submission(bytes, objects)?; + if pending.iter().any(|(command, _, _)| matches!(command, Validated::Render { .. })) { + self.context.note_render(); + } if super::timeout::select_submission(pending.iter().map(|(cmd, _, _)| cmd.long_job())) { for (command, _, _) in pending.iter_mut() { command.allow_long_job(); } } let mut guards = guards.into_iter(); - let aggregate = outputs.publish(self.context.clone())?; + let firmware_source = pending.len() == 1 + && matches!(&pending[0].0, Validated::Compute { .. }) + && !pending[0].2.has_outputs(); + let aggregate = outputs.publish(self.context.clone(), firmware_source)?; for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( @@ -429,6 +474,8 @@ impl crate::queue::Queue for Queue { } Ok(()) })(); + // Everything this ioctl publishes is visible; release what it pruned. + self.finished.clear(); if let Err(error) = result { self.context.status().record_if_device_loss(error); if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index fb6c77fbdf04f..696a1486eb0ec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -7,7 +7,7 @@ pub(crate) use observation::Observed; mod dependencies; mod memory; pub(crate) mod parked; -mod publication; +pub(super) mod publication; pub(crate) mod retirement; mod warm; @@ -46,6 +46,9 @@ pub(crate) struct RenderDependencies { pub(crate) tiling: [KickDependency; 2], pub(crate) tiling_count: usize, pub(crate) compute: Option, + /// Fragment kicks of other pairs' overlapping same-VM renders. + pub(crate) hazards: [KickDependency; crate::g17::hazard::PUBLICATION_BARRIERS], + pub(crate) hazard_count: usize, } impl RenderDependencies { @@ -55,6 +58,8 @@ impl RenderDependencies { tiling: [KickDependency::ZERO; 2], tiling_count: 0, compute: None, + hazards: [KickDependency::ZERO; crate::g17::hazard::PUBLICATION_BARRIERS], + hazard_count: 0, }; pub(crate) fn render(&self) -> &[KickDependency] { &self.render[..self.render_count] @@ -78,12 +83,13 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish( + /// Publishes `owners` in order with one pass over the QoS table. + fn qos_publish_pair( &mut self, - owner: qos::Owner, + owners: [qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result; + ) -> Result<[qos::Publication; 2]>; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; @@ -186,6 +192,9 @@ pub(crate) struct Pair { /// An owner change in flight: the previous scheduler page and the ordinal of /// the new owner's first command, until that command leaves the pair. switch: Option<(u64, u64)>, + /// Completion values of the two kicks of the last publication, kept with the configuration + /// cache: what each queue holds once that publication completed. + published_completions: Option<[u64; 2]>, } impl Pair { #[inline(never)] @@ -284,6 +293,7 @@ impl Pair { context_update: true, executed: 0, switch: None, + published_completions: None, })) } pub(crate) fn qids(&self) -> [u8; 2] { @@ -368,6 +378,7 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; + self.memory.prefetch_descriptors(ordinal); let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); @@ -484,7 +495,7 @@ impl Pair { struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, - fragment: [KickDependency; 3], + fragment: [KickDependency; 3 + crate::g17::hazard::PUBLICATION_BARRIERS], fragment_count: usize, } @@ -495,7 +506,7 @@ fn render_barriers( ) -> RenderBarriers { let mut tiling_barriers = [KickDependency::ZERO; 3]; let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_barriers = [own_tiling; 3 + crate::g17::hazard::PUBLICATION_BARRIERS]; let mut fragment_count = 1; for dependency in dependencies.render() { if fragment_stage { @@ -521,6 +532,11 @@ fn render_barriers( tiling_barriers[tiling_count] = compute; tiling_count += 1; } + // Only tile stores overlap: the tiling kick stays concurrent. + for hazard in &dependencies.hazards[..dependencies.hazard_count] { + fragment_barriers[fragment_count] = *hazard; + fragment_count += 1; + } RenderBarriers { tiling: tiling_barriers, tiling_count, @@ -637,10 +653,26 @@ impl Pair { } else { [None; 2] }; - let present = [ - publication::required(cache[0], configs[0]), - publication::required(cache[1], configs[1]), - ]; + // A covered queue holds the completion value of its newest kick. When that kick is the + // previous publication of this pair in this recovery epoch, the seed would reload the + // value already held, so the configuration record is needed only for its identity. + // Each record costs the firmware time before it starts the kick. + let held = self + .published_completions + .take() + .filter(|_| self.recovery_generation == epoch); + let present = [0, 1].map(|stage| { + let mut config = configs[stage]; + let reload = kicks[stage].prev().completion_value(payload.wrapping_sub(1)); + if config.completion_seed_valid != 0 + && self.installed[stage] + && held.is_some_and(|held| held[stage] == reload) + { + config.completion_seed_valid = 0; + config.completion_seed = 0; + } + publication::required(cache[stage], config) + }); let cursors = self.memory.graph.cursors()?; if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { return Err(EBUSY); @@ -790,10 +822,12 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - qos_publications[1] = - Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); - qos_publications[0] = - Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); + let [fragment, tiling] = host.qos_publish_pair( + [qos_owners[1], qos_owners[0]], + context.scheduler_va(), + policy, + )?; + qos_publications = [Some(tiling), Some(fragment)]; // A class change happens only at a rebind, after the pair drained. if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; @@ -840,6 +874,9 @@ impl Pair { } }; self.ordinal = next_ordinal; + if let Some(fragment) = KickDependency::new(qids[1], kicks[1]) { + packet.completion.note_render_publication(fragment); + } self.cursors = windows.map(|window| window.target()); for stage in 0..2 { if present[stage] { @@ -848,6 +885,7 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; + self.published_completions = Some(kicks.map(|kick| kick.completion_value(payload))); self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); @@ -941,6 +979,8 @@ impl Pair { host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; host.notify_render(priority)?; + crate::g17::trace::publish(crate::g17::trace::RENDER, self.qids()[1], self.ordinal as u32, + packet); host.note_submission() })(); if let Err(error) = publish { @@ -1073,6 +1113,28 @@ impl Pair { .front() .map(|active| active.packet.completion.spared()) } + /// A recovery record names one of this pair's queues and the completion + /// stamp of the group it interrupted, `(ordinal + 1) << 8` like the pair's + /// stamps. Without a stamp match the oldest published group is marked. + pub(crate) fn note_interrupted(&self, stamp: u32) -> recovery::Interrupted { + let sequence = stamp >> 8; + if let Some(active) = self + .active + .iter() + .filter(|active| active.outer_published) + .find(|active| (active.ticket.ordinal.wrapping_add(1) as u32) & 0xff_ffff == sequence) + { + active.packet.completion.note_interrupted(); + return recovery::Interrupted::Exact; + } + match self.active.iter().find(|active| active.outer_published) { + Some(active) => { + active.packet.completion.note_interrupted(); + recovery::Interrupted::Oldest + } + None => recovery::Interrupted::None, + } + } pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { for active in self.active.iter() { visit(active.packet.completion.status()); @@ -1101,6 +1163,7 @@ impl Pair { pub(crate) fn resync(&mut self) { self.configs = [None; 2]; + self.published_completions = None; for kick in &mut self.kicks { kick.clear_parent_after_recovery(); } @@ -1223,6 +1286,9 @@ impl Pair { front.qos_completed = true; front.flist_completed = true; } + crate::g17::recovery::note_retired(); + crate::g17::trace::complete(crate::g17::trace::RENDER, qids[1], ordinal as u32, + &front.packet, [timestamps[0], timestamps[3]]); defer(Deferred::Retired( front.packet.completion.clone(), Ok(timestamps), @@ -1254,6 +1320,9 @@ impl Pair { if self.quarantined || self.active.len() == 0 || self.replay.len() != 0 { return Ok(false); } + if let Some(progressed) = self.settle_interrupted_front(host, defer, grace)? { + return Ok(progressed); + } use crate::g17::replay::{self, Decision, Group}; let qids = self.qids(); let memory = &self.memory; @@ -1264,10 +1333,9 @@ impl Pair { let blamed = active.packet.completion.blamed(); let exhausted = active.packet.completion.replay_count() >= crate::g17::completion::MAX_REPLAYS; - // decide() stops at the first group that is not abandoned, an error included. - let abandoned = innocent - && !blamed - && (|| -> Result { + // decide() stops at the first group that is still live, an error included. + let (abandoned, ran, lost) = if innocent && !blamed { + (|| -> Result<(bool, bool, bool)> { let mut observer = Observer { memory, qids, @@ -1278,20 +1346,33 @@ impl Pair { let stamps = observer.stamps()?; let (_, state) = observer.host.epoch()?; // A torn sample is no proof; a later poll or the timeout looks again. - Ok(matches!( - retirement::abandoned(&active.ticket, snapshot, stamps, state, false), - Ok(true) - )) + let witness = |executed| { + matches!( + retirement::abandoned(&active.ticket, snapshot, stamps, state, executed), + Ok(true) + ) + }; + let unexecuted = witness(false); + // Interrupted work is run again only if it never started, and work that + // another queue waited on never: the wait ended when firmware retired it. + let lost = (active.packet.completion.interrupted() && !snapshot.untouched()) + || active.packet.completion.has_firmware_consumer(); + Ok((unexecuted && !lost, !unexecuted && witness(true), unexecuted && lost)) })() .unwrap_or_else(|error: Error| { failure = Some(error); - false - }); + (false, false, false) + }) + } else { + (false, false, false) + }; Group { innocent, blamed, abandoned, exhausted, + ran, + lost, } }); let decision = replay::decide(groups); @@ -1306,7 +1387,7 @@ impl Pair { front.abandoned_since = None; return Ok(false); } - Decision::Replay => {} + Decision::Replay | Decision::Fail => {} } if grace { let Some(since) = front.abandoned_since else { @@ -1317,6 +1398,9 @@ impl Pair { return Ok(false); } } + if decision == Decision::Fail { + return self.fail_dropped(host, defer); + } // decide() lets only the front be exhausted. An exhausted front fails, and its // successors, which consume its output, fail with it rather than run without it. let mut failing = self.active.front().is_some_and(|front| { @@ -1369,11 +1453,126 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; packet.reset_publication_for_replay(); + self.published_completions = None; self.replay.push(packet).map_err(|_| EIO)?; } Ok(true) } + /// Firmware retired every group of the pair after a restart, but some were dropped + /// unexecuted and cannot run again in order (see `replay::decide`). Every group fails now, + /// the ones that ran too: they consumed output the dropped ones never wrote. The work is + /// innocent, so it fails without poisoning its VM, with ENODATA, which its owner sees, + /// instead of waiting for the render timeout to fail it with ETIMEDOUT. + fn fail_dropped( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + pr_warn!( + "G17: render slot {}: a GPU recovery dropped groups that cannot run again in order; failing {} groups\n", + self.slot, + self.active.len() + ); + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + while let Some(front) = self.active.front_mut() { + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let completion = front.packet.completion.clone(); + completion.classify(recovery::Class::Spared); + defer(Deferred::Retired(completion, Err(ENODATA)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + Ok(true) + } + + /// A front group a restart interrupted after its tiler started, which firmware + /// then retired without writing its fragment end, may have run partly. It is + /// not replayed (blending or storage writes could apply twice) and fails now, + /// reported to its owner, instead of waiting out the render timeout. Returns + /// None when the front is not such a group. + fn settle_interrupted_front( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + grace: bool, + ) -> Result> { + use retirement::Memory; + let qids = self.qids(); + let Some(front) = self.active.front() else { + return Ok(None); + }; + if !front.outer_published || front.failed || !front.packet.completion.interrupted() { + return Ok(None); + } + let ordinal = front.ticket.ordinal; + let ticket = front.ticket; + let mut observer = Observer { + memory: &self.memory, + qids, + host: &mut *host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if snapshot.untouched() + || !matches!( + retirement::abandoned(&ticket, snapshot, stamps, state, false), + Ok(true) + ) + { + return Ok(None); + } + let now = crate::g17::now_ns(); + let front = self.active.front_mut().ok_or(EIO)?; + if grace { + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(Some(false)); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(Some(false)); + } + } + if !front.flist_completed { + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + pr_warn!( + "G17: render slot {}: group {} was interrupted by a GPU recovery after it started; failing it\n", + self.slot, + ordinal + ); + let packet = self.active.front().ok_or(EIO)?.packet.clone(); + defer(Deferred::Retired(packet.completion.clone(), Err(ECANCELED)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + Ok(Some(true)) + } + /// The pending replay cannot proceed (closed owner, cancellation or a /// publication refusal). Its successors cannot run without it and fail with /// it. ECANCELED leaves the VM usable; any other error is the ordinary @@ -1770,6 +1969,55 @@ impl Pair { } /// The scheduler page these queues last ran a command for, when the next /// publication for `context` changes the queues' owner. + /// The steady state in which a command may publish without the device + /// mutex: the graph is installed and registered at `priority`, it last ran + /// `context`'s scheduler, and nothing is replayed, settled, aliased or + /// switched on it. + pub(crate) fn direct_ready(&self, context: &Context, priority: u8) -> bool { + self.graph_started + && self.installed == [true; 2] + && self.registered == Some(priority) + && priority < 8 + && self.outer_started & (1u8 << priority) != 0 + && self.executed == context.scheduler_va() + && self.switch.is_none() + && self.owner.is_some() + && !self.quarantined + && self.terminal.is_none() + && self.pending_aliases.is_none() + && self.kick_aliases.is_none() + && self.prepared.is_none() + && self.replay.len() == 0 + && self.displaced.is_none() + && self.active.room() + } + /// The render prefix of `packet` on this pair, as the registry collects it + /// for an owner whose only pair this is. + pub(crate) fn own_dependencies(&self, owner: u64, packet: &Arc) -> Result { + let mut dependencies = RenderDependencies::EMPTY; + let mut frontier = crate::g17::dependency::Frontier::new(); + self.collect_dependencies(owner, packet.completion.status(), packet.order.wait_through[0], &mut frontier, false)?; + for dependency in frontier.as_slice() { + *dependencies.render.get_mut(dependencies.render_count).ok_or(EOVERFLOW)? = *dependency; + dependencies.render_count += 1; + } + if dependencies.render_count != 0 + && matches!(&packet.command, Validated::Render { pass, .. } if pass.fragment_barriers) + { + let mut tiling = crate::g17::dependency::Frontier::new(); + self.collect_tiling_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut tiling, + )?; + for dependency in tiling.as_slice() { + *dependencies.tiling.get_mut(dependencies.tiling_count).ok_or(EOVERFLOW)? = *dependency; + dependencies.tiling_count += 1; + } + } + Ok(dependencies) + } pub(crate) fn owner_switch_from(&self, context: &Context) -> Option { (self.executed != 0 && self.executed != context.scheduler_va()).then_some(self.executed) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index 53a0fa6e449ed..ab96b48de3e35 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -31,6 +31,7 @@ fn include( prefix: Option, packet: &Arc, word: u64, + compute: bool, ) -> Result { if !Arc::ptr_eq(packet.completion.status(), status) { return Ok(()); @@ -38,12 +39,15 @@ fn include( if !job::Order::contains(prefix, packet.order.sequence) { return Ok(()); } - frontier.include( - prefix, - packet.order.sequence, - job::fence_status(&packet.completion.scheduler_fence()), - word, - ) + let status = job::fence_status(&packet.completion.scheduler_fence()); + frontier.include(prefix, packet.order.sequence, status, word)?; + // A compute consumer's firmware wait ends once firmware retires this group, even unexecuted + // after a restart, so it may then read output the group never wrote. Render consumers of the + // same owner follow the group on its pair, and a replay keeps that order. + if compute && status == 0 { + packet.completion.note_firmware_consumer(); + } + Ok(()) } impl Dependencies { @@ -60,12 +64,13 @@ impl Dependencies { status: &Arc, prefix: Option, frontier: &mut Frontier, + compute: bool, ) -> Result { if self.owner != Some(owner) { return Ok(()); } for entry in self.entries.iter().flatten() { - include(frontier, status, prefix, &entry.packet, entry.word)?; + include(frontier, status, prefix, &entry.packet, entry.word, compute)?; } Ok(()) } @@ -80,7 +85,7 @@ impl Dependencies { return Ok(()); } for entry in self.entries.iter().flatten() { - include(frontier, status, prefix, &entry.packet, entry.ta_word)?; + include(frontier, status, prefix, &entry.packet, entry.ta_word, false)?; } Ok(()) } @@ -112,6 +117,7 @@ impl Pair { status: &Arc, prefix: Option, frontier: &mut Frontier, + compute: bool, ) -> Result { if self.owner != Some(owner) { return Ok(()); @@ -120,7 +126,7 @@ impl Pair { for entry in self.active.iter() { let word = KickDependency::new(qid, entry.kicks[1]).map_or(u64::MAX, KickDependency::word); - include(frontier, status, prefix, &entry.packet, word)?; + include(frontier, status, prefix, &entry.packet, word, compute)?; } Ok(()) } @@ -138,7 +144,7 @@ impl Pair { for entry in self.active.iter() { let word = KickDependency::new(qid, entry.kicks[0]).map_or(u64::MAX, KickDependency::word); - include(frontier, status, prefix, &entry.packet, word)?; + include(frontier, status, prefix, &entry.packet, word, false)?; } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 80ee0b871cc82..756f96e3fd048 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -537,6 +537,34 @@ const _: () = { }; impl Memory { + /// Requests the tiling descriptor slot and both queue configuration records of + /// `ordinal` for writing. The firmware accessed them for the command that used them + /// last, so the host no longer holds their lines; requesting them before the rest of + /// the preparation overlaps those transfers with it instead of stalling the writes one + /// line at a time. + pub(super) fn prefetch_descriptors(&self, ordinal: u64) { + let record = ordinal as usize % RECORDS; + let config = size_of::(); + let ranges = [ + (&self.graph.descriptors, Graph::offset(0, ordinal), size_of::()), + (&self.graph.config, CONFIG[0] + record * CONFIG_STRIDE, config), + (&self.graph.config, CONFIG[1] + record * CONFIG_STRIDE, config), + ]; + for (object, offset, size) in ranges { + let Ok(base) = object.pointer(offset, size) else { + continue; + }; + for line in (0..size).step_by(64) { + // SAFETY: A prefetch hint inside the retained mapping; it has no + // architectural effect on memory. + unsafe { + core::arch::asm!("prfm pstl1keep, [{0}]", in(reg) base.add(line), + options(nostack, preserves_flags, readonly)); + } + } + } + } + pub(super) fn write_descriptors( &mut self, args: &abi::RenderArgs<'_>, @@ -571,21 +599,10 @@ impl Memory { (&mut *tiling).write(args)?; let fragment = &mut *fragment; fragment.write(args)?; - self.registers.bytes_mut().fill(0); - for (index, array) in fragment.descriptor.arrays.iter().enumerate() { - for (entry, value) in array - .entries - .get(..(array.size & 0xffff) as usize) - .ok_or(EINVAL)? - .iter() - .enumerate() - { - self.registers.write( - index * size_of::() - + entry * size_of::(), - *value, - )?; - } + // Kick entries bind the arrays embedded in this descriptor + // (register_bindings); the client register page is never read. + for array in &fragment.descriptor.arrays { + array.entries.get(..(array.size & 0xffff) as usize).ok_or(EINVAL)?; } fragment.set_mcache( args.fragment.descriptor_va, diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 209ce0b8176ec..81d4306eda28c 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -252,6 +252,7 @@ impl ParkedPair { context_update: true, executed: self.executed, switch: None, + published_completions: None, })) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 9f4cb05b6bede..c1985c72f6ec1 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -19,7 +19,7 @@ fn identity(mut config: QueueConfig) -> QueueConfig { config.qos_update = 0; config } -pub(super) fn required(previous: Option, config: QueueConfig) -> bool { +pub(in crate::g17::queue) fn required(previous: Option, config: QueueConfig) -> bool { config.install != 0 || config.context_update != 0 || config.completion_seed_valid != 0 diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs index 24aeed6bcd0ec..00e4997ef3661 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -122,6 +122,11 @@ impl Snapshot { fn executed(self) -> bool { self.pass[0] != 0 && self.pass[1] != 0 } + + /// Neither the tiler start nor the fragment end was written: nothing ran. + pub(crate) fn untouched(self) -> bool { + self.pass == [0, 0] + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 89f8d33bdb80c..3bb9d20190b57 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -10,6 +10,7 @@ mod attribution; mod slot; mod safety; +pub(crate) use safety::{note_retired, since_retired}; pub(crate) use attribution::Sources; use attribution::Attribution; @@ -24,20 +25,34 @@ const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; const MAX_GUILTY_VMS: usize = 8; -const RECOVERY_POLLS: usize = 500; +/// One-millisecond ticks per handshake phase. The firmware finishes a restart only after the +/// work it interrupted has drained; with several clients running long ray-traversal jobs that +/// takes seconds, and giving up loses every client's device. Ten seconds matches the longest +/// execution allowance (`timeout::POLL_MS` plus renewals). +const RECOVERY_POLLS: usize = 10_000; -/// Per-QID classifications from the complete firmware recovery table. +/// Records in the firmware recovery table. +pub(super) const RECOVERY_RECORDS: usize = 256; + +/// Per-QID classifications from the firmware recovery table. /// Multiple records for a queue contribute all of their classification bits. #[derive(Default)] pub(super) struct Entries { guilty: u128, cancelled: u128, + /// Valid records, each naming one command that was in flight at the restart. + pub(super) listed: u32, + /// Listed commands found by their exact stamp, and by the oldest-work fallback. + pub(super) exact: u32, + pub(super) fallback: u32, } impl Entries { - pub(super) fn include(&mut self, flags: u32) { + /// Returns false at the first invalid record: the table is a packed list and + /// records after it are left over from an earlier restart. + pub(super) fn include(&mut self, flags: u32) -> bool { if flags & 1 == 0 { - return; + return false; } let qid = (flags >> 4) & 0x7f; let bit = 1u128 << qid; @@ -47,9 +62,22 @@ impl Entries { if flags & u32::from(CANCELLED) != 0 { self.cancelled |= bit; } + self.listed += 1; + true } } +/// How a listed recovery record was matched to retained work. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Interrupted { + /// No retained command of the queue. + None, + /// The command whose completion stamp equals the record's. + Exact, + /// No stamp matched; the queue's oldest unretired command was marked instead. + Oldest, +} + /// How an error completion affects its VM's sticky status and other jobs. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Class { @@ -71,10 +99,17 @@ pub(super) trait Queues { fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); /// Classify every in-flight completion; return QIDs of render slots with spared work. fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; + /// Mark the command a recovery record lists as interrupted. It may have + /// partly executed, so it must never report success; see + /// `Completion::note_interrupted`. + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> Interrupted; fn settle_blamed_render(&mut self, qid: u8, error: Error); fn settle_cancelled_renders(&mut self, qids: u128); fn rescan_compute(&mut self) -> Result; fn release_retired_compute(&mut self); + /// Ring every compute doorbell still withheld: a restart is a departure from ordinary + /// retirement, after which the firmware must see every published command. + fn announce_withheld(&mut self); } /// Live shared-memory operations. The handshake word uses aligned 32-bit accesses; transitions @@ -85,7 +120,9 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - fn entries(&self) -> Result; + /// One recovery record's flags and completion stamp, read while firmware + /// is halted in state 3. + fn entry(&self, index: usize) -> Result<(u32, u32)>; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -119,7 +156,9 @@ impl State { restarts: safety::Storm::new(), } } - pub(super) fn timeout_limit(&mut self, now: u64) -> bool { self.timeouts.note(now) } + pub(super) fn timeout_limit(&mut self, now: u64) -> bool { + self.timeouts.note_unless_progress(now, safety::retired()) + } pub(super) fn restart_limit(&mut self, now: u64) -> bool { self.restarts.note(now) } pub(super) fn generation(&self) -> u64 { self.accepted @@ -192,6 +231,8 @@ pub(super) fn service( } } if !ready { + pr_warn!("G17: recovery gen {} did not reach state 1 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // Recheck ownership after the off-CPU waits, before any handshake mutation. @@ -217,17 +258,31 @@ pub(super) fn service( host.wait_tick(); } if !recovered { + pr_warn!("G17: recovery gen {} did not reach state 3 (state {:?})\n", + request.generation, host.recovery_state()); return Err(ETIMEDOUT); } // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let entries = host.entries().unwrap_or_default(); + // Every listed command was in flight when firmware stopped. Firmware may + // still retire it with stamps and records that look like success, so it is + // marked before firmware resumes and never reports success. + let entries = read_entries(host); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; + let (listed, exact, fallback) = (entries.listed, entries.exact, entries.fallback); let (cancelled, blame) = classify(host, entries, request.blamed, &mut sources); + if listed != 0 { + pr_warn!( + "G17: GPU recovery: {} commands were in flight ({} matched by stamp, {} by queue); they fail instead of completing\n", + listed, + exact, + fallback + ); + } if let Some(qid) = request.blamed { pr_warn!( "G17: GPU recovery (reason {:?}): firmware blamed queue {}, {}\n", @@ -244,9 +299,30 @@ pub(super) fn service( } let scan_error = host.rescan_compute().err(); host.release_retired_compute(); + host.announce_withheld(); Ok(ServiceResult { scan_error, target_error }) } +/// Walks the packed recovery table up to its first invalid record and marks +/// every listed command interrupted. A failed read keeps what was read so far. +fn read_entries(host: &mut H) -> Entries { + let mut entries = Entries::default(); + for index in 0..RECOVERY_RECORDS { + let Ok((flags, stamp)) = host.entry(index) else { + break; + }; + if !entries.include(flags) { + break; + } + match host.note_interrupted(((flags >> 4) & 0x7f) as u8, stamp) { + Interrupted::Exact => entries.exact += 1, + Interrupted::Oldest => entries.fallback += 1, + Interrupted::None => {} + } + } + entries +} + /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. diff --git a/drivers/gpu/drm/asahi/g17/recovery/safety.rs b/drivers/gpu/drm/asahi/g17/recovery/safety.rs index c30e45e22579a..082416f0cb9a6 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/safety.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/safety.rs @@ -2,17 +2,52 @@ //! Fail closed on repeated watchdog/restart traffic, even across client loss. +use core::sync::atomic::{AtomicU64, Ordering}; + const WINDOW_NS: u64 = 30_000_000_000; const LIMIT: u8 = 8; +/// Commands retired successfully. A watchdog storm means the GPU stopped making progress; +/// timeouts of overloaded clients while other work keeps retiring are not one. +static RETIRED: AtomicU64 = AtomicU64::new(0); +/// [`crate::g17::now_ns`] of the latest successful retirement, zero before the first. +static LAST_RETIRED_NS: AtomicU64 = AtomicU64::new(0); + +/// Called for every command that retires without error. +pub(crate) fn note_retired() { + RETIRED.fetch_add(1, Ordering::Relaxed); + LAST_RETIRED_NS.store(crate::g17::now_ns(), Ordering::Relaxed); +} + +pub(super) fn retired() -> u64 { + RETIRED.load(Ordering::Relaxed) +} + +/// Nanoseconds since any command last retired successfully, or `None` before the first. +pub(crate) fn since_retired(now: u64) -> Option { + match LAST_RETIRED_NS.load(Ordering::Relaxed) { + 0 => None, + last => Some(now.saturating_sub(last)), + } +} + pub(super) struct Storm { start: Option, count: u8, stopped: bool, + progress: u64, } impl Storm { pub(super) const fn new() -> Self { - Self { start: None, count: 0, stopped: false } + Self { start: None, count: 0, stopped: false, progress: 0 } + } + /// Like [`Self::note`], but work retired since the previous event starts a new window. + pub(super) fn note_unless_progress(&mut self, now: u64, progress: u64) -> bool { + if !self.stopped && progress != self.progress { + self.progress = progress; + self.start = None; + } + self.note(now) } pub(super) fn note(&mut self, now: u64) -> bool { if self.stopped { return true; } @@ -74,6 +109,17 @@ mod tests { for _ in 0..1024 { assert!(storm.note(u64::MAX)); } } #[test] + fn recovery_table_ends_at_its_first_invalid_record() { + use crate::g17::recovery::Entries; + let mut entries = Entries::default(); + assert!(entries.include(1 | (12 << 4))); + assert!(entries.include(1 | 2 | (17 << 4))); + assert!(!entries.include(0)); + assert_eq!(entries.listed, 2); + assert_eq!(entries.guilty, 1u128 << 17); + assert_eq!(entries.cancelled, 0); + } + #[test] fn replay_dependency_checks_never_wait_for_event_service() { use crate::g17::dependency::replay_status; assert_eq!(replay_status(0), Err(EBUSY)); diff --git a/drivers/gpu/drm/asahi/g17/replay.rs b/drivers/gpu/drm/asahi/g17/replay.rs index 94daa5ec0dfeb..7668871376bad 100644 --- a/drivers/gpu/drm/asahi/g17/replay.rs +++ b/drivers/gpu/drm/asahi/g17/replay.rs @@ -16,6 +16,12 @@ pub(crate) struct Group { pub(crate) abandoned: bool, /// Already republished the maximum number of times. pub(crate) exhausted: bool, + /// Retired by firmware after executing in full. + pub(crate) ran: bool, + /// Retired by firmware unexecuted, but never to run again: a recovery interrupted it after + /// its tiler started, so it may have run partly, or a command of another queue waited on it, + /// and that wait ended when firmware retired the group, before its output existed. + pub(crate) lost: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -25,26 +31,54 @@ pub(crate) enum Decision { Wait, /// Never replay this set; the render timeout settles it. Refuse, + /// Firmware has retired every group, but a dropped group cannot run again in order: a later + /// group already ran, or the dropped group had started or had a consumer. Fail the set now. + Fail, } /// Groups are read in order and the first one that decides stops the walk, so a caller can /// observe the abandonment lazily. An exhausted front is replayed: the replay fails it, and its /// successors, which consume its output, with it. An exhausted successor would break the order. +/// +/// After a restart the firmware retires the groups it had started on a pair without running +/// them, while groups it had not yet started run as usual. Once such a later group has run, a +/// replay of the dropped ones would run them after it; a group that had started cannot run +/// twice; and a group another queue waited on may already have been read before it ran. None of +/// these is replayed: the whole set fails at once instead of waiting for the render timeout. A +/// group that ran ahead of every dropped one is left to in-order retirement. pub(crate) fn decide(groups: impl Iterator) -> Decision { let mut front_exhausted = None; let mut successor_exhausted = false; + let mut dropped = false; + let mut unordered = false; for group in groups { if !group.innocent || group.blamed { return Decision::Refuse; } + if group.ran { + if !dropped { + return Decision::Wait; + } + unordered = true; + continue; + } + if group.lost { + dropped = true; + unordered = true; + continue; + } if !group.abandoned { return Decision::Wait; } + dropped = true; match front_exhausted { None => front_exhausted = Some(group.exhausted), Some(_) => successor_exhausted |= group.exhausted, } } + if unordered { + return Decision::Fail; + } match front_exhausted { Some(true) => Decision::Replay, Some(false) if !successor_exhausted => Decision::Replay, @@ -56,10 +90,13 @@ pub(crate) fn decide(groups: impl Iterator) -> Decision { mod tests { use super::*; fn group(innocent: bool, abandoned: bool, exhausted: bool) -> Group { - Group { innocent, blamed: false, abandoned, exhausted } + Group { innocent, blamed: false, abandoned, exhausted, ran: false, lost: false } } - const OK: Group = Group { innocent: true, blamed: false, abandoned: true, exhausted: false }; - const BLAMED: Group = Group { innocent: true, blamed: true, abandoned: true, exhausted: false }; + const OK: Group = + Group { innocent: true, blamed: false, abandoned: true, exhausted: false, ran: false, lost: false }; + const BLAMED: Group = Group { blamed: true, ..OK }; + const RAN: Group = Group { abandoned: false, ran: true, ..OK }; + const LOST: Group = Group { abandoned: false, lost: true, ..OK }; #[test] fn every_group_abandoned_and_innocent_is_replayed() { assert_eq!(decide([OK].into_iter()), Decision::Replay); @@ -106,4 +143,28 @@ mod tests { Decision::Replay ); } + #[test] + fn a_group_that_ran_after_dropped_ones_fails_the_set() { + // Seen after restarts: the started groups were dropped, the queued ones ran. + assert_eq!(decide([OK, OK, RAN, RAN].into_iter()), Decision::Fail); + assert_eq!(decide([OK, RAN, OK].into_iter()), Decision::Fail); + // Also with an exhausted front, which a replay would only fail later. + assert_eq!(decide([group(true, true, true), RAN].into_iter()), Decision::Fail); + } + #[test] + fn a_group_that_ran_first_is_left_to_retirement() { + assert_eq!(decide([RAN].into_iter()), Decision::Wait); + assert_eq!(decide([RAN, OK].into_iter()), Decision::Wait); + } + #[test] + fn a_started_dropped_group_fails_the_set() { + assert_eq!(decide([OK, LOST].into_iter()), Decision::Fail); + assert_eq!(decide([LOST, OK].into_iter()), Decision::Fail); + } + #[test] + fn failing_still_waits_for_live_groups_and_refuses_guilty_ones() { + assert_eq!(decide([OK, RAN, group(true, false, false)].into_iter()), Decision::Wait); + assert_eq!(decide([OK, RAN, group(false, true, false)].into_iter()), Decision::Refuse); + assert_eq!(decide([LOST, BLAMED].into_iter()), Decision::Refuse); + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6f4a5355d6eb3..e04295d576606 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -31,9 +31,32 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; -const RECOVERY_SETTLE_POLLS: usize = 200; const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; +/// Pending admission keeps exactly retired compute backing available for reuse. +/// The guard spans retries and disappears on publication, cancellation or error. +/// It borrows Shared, so the accounting cannot retain the device after teardown. +struct ComputeDemand<'a>(&'a Arc); + +impl<'a> ComputeDemand<'a> { + fn new(shared: &'a Arc) -> Self { + // Each admitted logical queue has at most one scheduler publisher. + // The execution-context bound is far below the counter's range. + shared.compute_demand.fetch_add(1, Ordering::AcqRel); + Self(shared) + } +} + +impl Drop for ComputeDemand<'_> { + fn drop(&mut self) { + if self.0.compute_demand.fetch_sub(1, Ordering::AcqRel) == 1 { + // Reinstate normal idle retention even if the last publisher was + // cancelled before a kick could produce another event. + self.0.queue_events(); + } + } +} + /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. #[derive(Clone, Copy)] @@ -45,13 +68,103 @@ pub(super) struct ComputeReservation { struct ComputeEntry { reservation: ComputeReservation, - queue: Option>, + queue: ComputeSlot, +} + +/// A physical compute queue behind its own lock. Publication into a bound queue takes only +/// this lock; device-mutex holders take it to observe or change the queue (device mutex -> +/// cell). A logical queue may keep the cell after the registry gave the queue up: the cell is +/// then empty, so the cached reference never extends any firmware-visible lifetime. +#[pin_data] +pub(super) struct ComputeCell { + #[pin] + queue: kernel::sync::Mutex>>, +} + +impl ComputeCell { + fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + queue <- kernel::new_mutex!(None, "g17::ComputeCell::queue"), + }), + GFP_KERNEL, + ) + } + + /// The installed queue, locked; None once the registry gave it up. + pub(super) fn lock(&self) -> Option> { + QueueGuard::new(self.queue.lock()) + } +} + +/// A locked installed queue. +pub(super) struct QueueGuard<'a> { + queue: core::ptr::NonNull, + _guard: kernel::sync::MutexGuard<'a, Option>>, +} + +impl<'a> QueueGuard<'a> { + fn new(mut guard: kernel::sync::MutexGuard<'a, Option>>) -> Option { + let queue = core::ptr::NonNull::from((*guard).as_deref_mut()?); + Some(Self { queue, _guard: guard }) + } +} + +impl core::ops::Deref for QueueGuard<'_> { + type Target = compute::Queue; + fn deref(&self) -> &compute::Queue { + // SAFETY: The boxed queue cannot be taken or dropped while the guard holds its lock. + unsafe { self.queue.as_ref() } + } +} + +impl core::ops::DerefMut for QueueGuard<'_> { + fn deref_mut(&mut self) -> &mut compute::Queue { + // SAFETY: As for `deref`; the lock makes this the only reference. + unsafe { self.queue.as_mut() } + } +} + +/// The registry's handle on one compute cell. Dropping it drops the queue in the cell, so a +/// cached cell never keeps a removed queue (or its QID's aliases) alive. +struct ComputeSlot(Arc); + +impl ComputeSlot { + fn new() -> Result { + Ok(Self(ComputeCell::new()?)) + } + fn as_deref(&self) -> Option> { + self.0.lock() + } + fn as_deref_mut(&mut self) -> Option> { + self.0.lock() + } + fn is_some(&self) -> bool { + self.0.queue.lock().is_some() + } + fn is_none(&self) -> bool { + !self.is_some() + } + /// Gives the queue up to the caller, which drops it outside the cell lock. + fn take(&mut self) -> Option> { + self.0.queue.lock().take() + } + fn put(&mut self, queue: Option>) { + let old = core::mem::replace(&mut *self.0.queue.lock(), queue); + drop(old); + } +} + +impl Drop for ComputeSlot { + fn drop(&mut self) { + let queue = self.0.queue.lock().take(); + drop(queue); + } } pub(super) struct Registry { pub(super) qids: kick::QueueIds, pub(super) buffers: BufferIds, - pub(super) accounting: qos::Accounting, pub(super) metrics: Arc, pub(super) qos_ids: Arc, pub(super) admission: Arc, @@ -62,6 +175,10 @@ pub(super) struct Registry { compute_scheduler_building: bool, replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, + /// Completion-ring slot where the next compute scan starts. + completion_cursor: usize, + /// Control word of each completion-ring slot as of the last scan that read it. + completion_seen: KVVec, worker_deferred: Option, teardown: super::teardown::Pending, } @@ -69,21 +186,34 @@ pub(super) struct Registry { impl Registry { /// Initialize inside the firmware allocation instead of staging the whole /// registry on the probe stack. No additional allocation is required. + #[cfg(CONFIG_KUNIT)] pub(super) fn new( clusters: u32, deflake: crate::g17::fw::render::Deflake, descriptor_flags: [u32; 2], variant: &'static crate::g17::fw::Variant, + ) -> impl Init { + Self::with_waiters(clusters, deflake, descriptor_flags, variant, None, None) + } + + /// `queues` wakes logical-queue creation when a logical queue slot is returned; + /// `renders` wakes render publication when a render pair or lease returns. + pub(super) fn with_waiters( + clusters: u32, + deflake: crate::g17::fw::render::Deflake, + descriptor_flags: [u32; 2], + variant: &'static crate::g17::fw::Variant, + queues: Option>, + renders: Option>, ) -> impl Init { kernel::try_init!(Self { qids <- kick::QueueIds::new(variant.qid_floor).chain(|qids| { qids.reserve_initial_render() }), buffers <- BufferIds::new(), - accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, - admission: admission::Pool::new(kick::queue_limit(variant.qid_floor))?, + admission: admission::Pool::with_waiters(kick::queue_limit(variant.qid_floor), queues, renders)?, render: render::RenderState::new(clusters, deflake, descriptor_flags)?, identities: queue::Identities::new(), compute: { @@ -97,12 +227,21 @@ impl Registry { compute_scheduler_building: false, replay_outer_wait: None, observations: compute::retirement::Batch::new()?, + completion_cursor: 0, + completion_seen: { + let slots = super::initdata::COMPUTE_COMPLETION_SLOTS; + let mut seen = KVVec::with_capacity(slots, GFP_KERNEL)?; + for _ in 0..slots { + seen.push_within_capacity(0).map_err(|_| ENOMEM)?; + } + seen + }, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, }) } - pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { + pub(super) fn compute(&self, owner: u64) -> Option> { self.compute .iter() .flatten() @@ -110,12 +249,16 @@ impl Registry { .find(|queue| queue.owner() == Some(owner)) } - pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { - self.compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.owner() == Some(owner)) + pub(super) fn compute_mut(&mut self, owner: u64) -> Option> { + self.compute(owner) + } + + /// The cell of `owner`'s bound queue, which its publisher may lock without the device mutex. + pub(super) fn compute_cell(&self, owner: u64) -> Option> { + self.compute.iter().flatten().find_map(|entry| { + entry.queue.as_deref()?.owner().filter(|bound| *bound == owner)?; + Some(entry.queue.0.clone()) + }) } /// An installed idle queue keeps its original QID and backing. Mapping a @@ -131,15 +274,80 @@ impl Registry { Rebind::Fresh }); } - // Physical channels are a device budget, not a permanent entitlement - // of each open logical queue. Only exact idle retirement can yield one. - for queue in self.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; + // A binding persists for its logical queue's lifetime while physical + // QIDs allow: take a channel of a closed owner, then a fresh QID, and + // only then the least recently idle channel of one open owner. + if let Some(rebind) = self.bind_reusable(owner, context, required, admission)? { + return Ok(rebind); } + if self.qids.compute_fits(self.admission.queues() as usize) { + return Ok(Rebind::Fresh); + } + let teardown = &self.teardown; + let idle = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.shareable()) + .filter(|queue| !queue.context().is_some_and(|bound| teardown.releasing(bound))) + .map(|queue| ((!queue.pool_fits(required), queue.last_published()), queue.qid())) + .min_by_key(|(key, _)| *key) + .map(|(_, qid)| qid); + let Some(qid) = idle else { return Err(ENOSPC); }; + admission.check()?; + let shared = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EIO)? + .share_idle()?; + if !shared { + return Err(ENOSPC); + } + Ok(self.bind_reusable(owner, context, required, admission)?.unwrap_or(Rebind::Fresh)) + } + + /// Whether `owner` can bind a channel now, without changing any binding. Owner 0 asks + /// only whether some channel is free for a new owner. + fn compute_readiness(&self, owner: u64, context: Option<&Arc>, required: usize) -> Readiness { + if let (Some(context), Some(queue)) = (context, self.compute(owner).filter(|_| owner != 0)) { + return if queue.spared_pending() { + Readiness::Progress + } else if !matches!(queue.matches(owner, queue.qid(), context), Ok(true)) { + // Publication reports the mismatch. + Readiness::Ready + } else if queue.pool_fits(required) || !queue.growing() { + Readiness::Ready + } else { + Readiness::Progress + }; + } + let teardown = &self.teardown; + let available = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + (queue.reusable() || queue.shareable() || queue.can_rearm()) + && !queue.context().is_some_and(|bound| teardown.releasing(bound)) + }); + if available || self.qids.compute_fits(self.admission.queues() as usize) { + Readiness::Ready + } else { + Readiness::Channel + } + } + + /// Binds `owner` to a reusable channel whose last owner released it, or + /// names a vacant one to re-arm. None if no channel is reusable. + fn bind_reusable(&mut self, owner: u64, context: &Arc, required: usize, admission: &Admission<'_>) -> Result> { let mut vacant = None; let teardown = &self.teardown; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -159,9 +367,9 @@ impl Registry { } if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; - return Ok(Rebind::Done); + return Ok(Some(Rebind::Done)); } - Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) + Ok(vacant.map(Rebind::Rearm)) } /// Installs fresh USC backing into the vacant queue `qid`; the caller writes @@ -172,7 +380,7 @@ impl Registry { pages: crate::g17::freelist::Pages, ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -184,7 +392,7 @@ impl Registry { pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { let teardown = &self.teardown; - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -252,7 +460,7 @@ impl Registry { let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { - let queue = self + let mut queue = self .compute .iter_mut() .flatten() @@ -282,7 +490,7 @@ impl Registry { let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); let mut freed = 0; - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -313,7 +521,7 @@ impl Registry { || self.compute.iter().flatten().any(|entry| { entry .queue - .as_ref() + .as_deref() .map_or(entry.reservation.owner == owner, |queue| { queue.owner() == Some(owner) }) @@ -328,14 +536,15 @@ impl Registry { .ok_or_else(|| self.compute_admission_error( crate::g17::capacity::refused(crate::g17::capacity::Site::ComputeSlots)))?; let identity = self.identities.reserve(1)?.start; + let queues = self.admission.queues() as usize; let id = self .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID)) + .reserve_for(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID), queues) .map_err(|error| self.compute_admission_error(error))?; let reservation = ComputeReservation { slot, id, owner }; self.compute[slot] = Some(ComputeEntry { reservation, - queue: None, + queue: ComputeSlot::new()?, }); Ok(reservation) } @@ -367,7 +576,7 @@ impl Registry { return Err(EIO); } let entry = self.reserved(reservation)?; - entry.queue = queue.take(); + entry.queue.put(queue.take()); Ok(()) } @@ -382,13 +591,14 @@ impl Registry { let Some(slot) = self.compute.iter().position(|entry| { entry .as_ref() - .and_then(|entry| entry.queue.as_ref()) + .and_then(|entry| entry.queue.as_deref()) .is_some_and(|queue| queue.owner() == Some(owner)) }) else { return Ok(()); }; let entry = self.compute[slot].as_mut().ok_or(EIO)?; - if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { + let unpublished = entry.queue.as_deref_mut().ok_or(EIO)?.release_owner()?; + if unpublished { self.qids.cancel_unpublished(entry.reservation.id)?; self.compute[slot] = None; } @@ -396,6 +606,16 @@ impl Registry { } } +/// What a compute packet still lacks before publication. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(super) enum Readiness { + Ready, + /// A physical channel: every one is busy and no QID fits. + Channel, + /// Retirement-bound capacity or reopened publication. + Progress, +} + /// Outcome of offering an owner a retained compute queue. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(super) enum Rebind { @@ -436,12 +656,28 @@ mod uninstalled_cleanup_tests { /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. +#[pin_data] pub(super) struct Backend { shared: Arc, pub(super) context: Arc, owner: u64, backlog: core::sync::atomic::AtomicU32, + compute_backlog: core::sync::atomic::AtomicU32, closed: core::sync::atomic::AtomicBool, + /// The cell of the compute queue this owner last published to under the device mutex, + /// with the shared compute objects mapped into its VM. Direct publication revalidates the + /// binding under the cell lock; a stale cell only sends the packet back to that path. + #[pin] + direct: kernel::sync::Mutex>>, + /// The channel a prepared packet is about to publish into. Its claim keeps the idle + /// channel from being shared away between prepare_job and run_job. + #[pin] + claimed: kernel::sync::Mutex>>, + /// Render pair admission, shared by every logical queue of the session. + admission: Arc, + /// The render pair this owner last published to without the device mutex + /// (zero for none): a hint, revalidated by every lend. + direct_slot: core::sync::atomic::AtomicU32, } impl Backend { @@ -459,6 +695,7 @@ impl Backend { const RETAINED_CONTEXT_WAIT_NS: u64 = 4 * WORK_RING_WAIT_NS; let started = now_ns(); let (lease, uat, pool, qos, execution, slot) = loop { + let observed = shared.queues.observe(); let lease = shared.preparations.enter(true)?; let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -466,6 +703,11 @@ impl Backend { return Err(EIO); } let slot = firmware.queues.admission.reserve_queue()?; + // Installed compute channels narrow render's QIDs: every open + // logical queue keeps a dedicated render pair within reach. + if !firmware.queues.qids.render_fits(firmware.queues.admission.queues() as usize) { + return Err(super::capacity::refused(super::capacity::Site::LogicalQueues)); + } match firmware.uat.new_execution_context(vm) { Ok(execution) => { break ( @@ -493,7 +735,10 @@ impl Backend { return Err(ERESTARTSYS); } shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + // A closing queue returns its context when its release + // is reclaimed; that reclaim wakes this creator. + let left = RETAINED_CONTEXT_WAIT_NS.saturating_sub(now_ns().saturating_sub(started)); + shared.queues.block(observed, left.div_ceil(1_000_000) as u32, true)?; } Err(error) => return Err(error), } @@ -506,7 +751,7 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, + Context::new(&alloc, &pool, &qos, execution, owner_pid, policy, shared.renders.clone())?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -514,15 +759,21 @@ impl Backend { return Err(EAGAIN); } drop(state); + let admission = slot.pool().clone(); Ok(( - Arc::new( - Self { + Arc::pin_init( + pin_init!(Self { shared: shared.clone(), context, owner, backlog: core::sync::atomic::AtomicU32::new(0), + compute_backlog: core::sync::atomic::AtomicU32::new(0), closed: core::sync::atomic::AtomicBool::new(false), - }, + direct <- kernel::new_mutex!(None, "g17::Backend::direct"), + claimed <- kernel::new_mutex!(None, "g17::Backend::claimed"), + admission, + direct_slot: core::sync::atomic::AtomicU32::new(0), + }), GFP_KERNEL, )?, slot, @@ -592,7 +843,7 @@ impl Backend { .queues .compute_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); drop(lease); if result != Err(EAGAIN) { return result; @@ -652,7 +903,7 @@ impl Backend { .queues .compute_scheduler_building = false; } - self.shared.changed.notify_all(); + self.shared.note_changed(); result } @@ -667,11 +918,14 @@ impl Backend { drop(queue); let mut state = self.shared.state.lock(); let firmware = state.as_deref_mut().ok_or(ENODEV)?; - firmware.queues.cancel_compute(reservation) + firmware.queues.cancel_compute(reservation)?; + drop(state); + // The slot and its never-published QID are free again. + self.shared.channels.notify(); + Ok(()) } fn grow_compute(&self, required: usize, admission: &Admission<'_>) -> Result { - use kernel::time::{delay::fsleep, Delta}; admission.check()?; { let state = self.shared.state.lock(); @@ -689,6 +943,9 @@ impl Backend { let mut deferred = DeferredBatch::publication()?; let mut growth = None; let result = (|| loop { + // Growth waits for this queue's retirement and for firmware to consume the pool + // release; both are progress the event worker reports. + let observed = self.shared.progress.observe(); let result = (|| { let mut state = self.shared.state.lock(); admission.check()?; @@ -696,7 +953,7 @@ impl Backend { if !lease.is_current() { return Err(EAGAIN); } if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } firmware.scan_compute(None, &mut deferred)?; - let queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; + let mut queue = firmware.queues.compute_mut(self.owner).ok_or(EAGAIN)?; if queue.pool_fits(required) { return Ok(None); } if growth.is_none() { let notify = queue.begin_growth(&firmware.init, required)?; @@ -714,9 +971,9 @@ impl Backend { match result { Ok(old) => { drop(old); return Ok(()); } Err(error) if error == EAGAIN && lease.is_current() => { - self.shared.queue_events(); - fsleep(Delta::from_millis(1)); - } + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; + } Err(error) => return Err(error), } })(); @@ -727,8 +984,9 @@ impl Backend { if let Some(firmware) = state.as_deref_mut() { for index in 0..firmware.queues.compute.len() { let Some(entry) = firmware.queues.compute[index].as_mut() else { continue; }; - let Some(queue) = entry.queue.as_deref_mut() else { continue; }; - if queue.abort_growth(token)? == Some(true) { + let Some(aborted) = entry.queue.as_deref_mut() + .map(|mut queue| queue.abort_growth(token)).transpose()? else { continue; }; + if aborted == Some(true) { // Keep the empty entry and QID reserved until all // graph aliases have been destroyed outside the lock. retired = entry.queue.take().map(|queue| (entry.reservation, queue)); @@ -762,9 +1020,11 @@ impl Backend { } loop { admission.check()?; + // Sampled before any capacity check of this attempt. + let observed = self.shared.channels.observe(); let lease = self.shared.preparations.enter_until(admission)?; let dev = self.shared.drm()?; - let (uat, reservation) = { + let (uat, objects, reservation) = { let mut state = self.shared.state.lock(); admission.check()?; let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -780,7 +1040,7 @@ impl Backend { Err(error) if error == ENOSPC => { drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -788,16 +1048,17 @@ impl Backend { match rebound { Rebind::Done => return Ok(()), Rebind::Grow => { drop(state); drop(lease); return self.grow_compute(required, admission); } - Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), + Rebind::Rearm(qid) => (firmware.uat.clone(), firmware.pool.clone(), Err(qid)), Rebind::Fresh => { let reservation = match firmware.queues.reserve_compute(self.owner) { Ok(reservation) => reservation, Err(error) if error == EAGAIN => { - // The event worker owns release observation and - // drops vacated aliases/backing off this mutex. + // All physical channels are occupied. Retain their + // USC backing for exact-retirement handoff; releasing + // it cannot create a QID and forces fresh allocation. drop(state); drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, false)?; continue; } Err(error) if error == ENOSPC => { @@ -811,7 +1072,7 @@ impl Backend { } Err(error) => return Err(error), }; - (firmware.uat.clone(), Ok(reservation)) + (firmware.uat.clone(), firmware.pool.clone(), Ok(reservation)) } } }; @@ -826,12 +1087,12 @@ impl Backend { // queue may have been taken or re-armed by another owner since. let built = (|| -> Result<_> { let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; - Ok((pages, compute::DescriptorRing::new(&alloc)?)) + Ok((pages, compute::DescriptorRing::new(&alloc, &objects)?)) })(); let (pages, ring) = match built { Err(error) if error == ENOSPC => { drop(lease); - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, true)?; continue; } result => result?, @@ -881,6 +1142,7 @@ impl Backend { self.compute_scheduler(&alloc, &lease, admission)?; let queue = compute::Queue::new( &alloc, + &objects, backing, self.owner, self.context.clone(), @@ -920,7 +1182,7 @@ impl Backend { } drop(lease); if result == Err(ENOSPC) { - self.wait_compute_capacity(admission)?; + self.wait_channels(observed, admission, true)?; continue; } if result != Err(EAGAIN) { return result; } @@ -932,6 +1194,7 @@ impl Backend { /// The physical namespace bounds the batch independently of open logical queues. /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { + has_source_completions: bool, values: KVVec, /// Exited owners detached from idle retained compute queues and the backing /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. @@ -967,6 +1230,7 @@ impl DeferredBatch { 0 }; Ok(Self { + has_source_completions: false, values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, @@ -1020,10 +1284,18 @@ impl DeferredBatch { // it. Expose terminal VM status now, before off-lock fence callbacks. // Recovery-spared and cancelled commands never poison their VM. if let Deferred::FailedOwned(completion, error) = &value { - if *error != ECANCELED && !completion.spared() { + if completion.poisons(*error) { completion.status().record(*error); } } + match &value { + Deferred::Retired(completion, Err(error)) | Deferred::Drained(completion, Err(error)) + | Deferred::FailedOwned(completion, error) | Deferred::Unpublished(completion, error) => + { completion.note_input_error(*error); }, + Deferred::Retired(completion, Ok(_)) | Deferred::Drained(completion, Ok(_)) => + completion.note_source_retired(), + _ => {}, + } self.values.push_within_capacity(value).map_err(|_| EIO)?; // A later publisher can borrow this pair before finish signals its old // fence. Publish the exact completion's witness only after collection @@ -1035,11 +1307,33 @@ impl DeferredBatch { ) = self.values.last() { completion.note_retirement_queued(); + self.has_source_completions |= completion.sources().is_some(); } Ok(()) } + /// Completions collected so far. + pub(super) fn pending(&self) -> usize { + self.values.len() + } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { + use super::completion::Deferred; + if self.has_source_completions { + for _ in 0..self.values.len() { + let mut changed = false; + for value in &self.values { + if let Deferred::Retired(completion, Ok(_)) + | Deferred::Drained(completion, Ok(_)) = value + { + if let Err(error) = completion.check_source_inputs() { + changed |= completion.note_input_error(error); + } + } + } + if !changed { break; } + } + } + self.has_source_completions = false; for value in self.values.drain_all() { value.finish(); } @@ -1060,15 +1354,15 @@ impl DeferredBatch { struct ComputeHost<'a> { init: &'a super::initdata::InitData, - primary: &'a mut super::Coprocessor, + port: &'a super::port::Port, + outer: Option>, effort: &'a mut super::power::Effort, qids: &'a mut kick::QueueIds, - accounting: &'a mut qos::Accounting, generation: u64, } impl compute::Host for ComputeHost<'_> { fn next_compute_publication(&mut self) -> Result { - self.qids.next_publication() + Ok(self.port.next_publication()) } fn epoch(&self) -> Result<(u64, u32)> { Ok(( @@ -1085,54 +1379,157 @@ impl compute::Host for ComputeHost<'_> { scheduler: u64, policy: super::fw::queue::Policy, ) -> Result { - self.accounting.publish( - &qos::View::new(self.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + self.port.qos_publish(owner, scheduler, policy) } fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.accounting - .cancel(&qos::View::new(self.init.qos()?)?, publication) + self.port.qos_cancel(publication) } fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(priority, slot).map(|_| ()) + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) } - fn notify(&mut self, message: u64) -> Result { - self.primary.notify(message) + fn notify_compute(&mut self, priority: u8, message: u64) -> Result { + self.port.notify_compute(priority, message) + } + fn compute_written(&self, priority: u8) -> u64 { + self.port.compute_written(priority) } fn note_submission(&mut self) -> Result { self.effort.submit(); - super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) + super::Firmware::apply_shared_effort(self.init, self.effort, self.port, super::power::FULL_EFFORT) + } +} + +/// Publication into a bound, installed queue through the port alone. The caller checked the +/// recovery state, full effort and the queue's binding; first installation never comes here. +struct DirectHost<'a> { + port: &'a super::port::Port, + outer: Option>, + generation: u64, +} +impl compute::Host for DirectHost<'_> { + fn next_compute_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn prepare_compute_shared(&mut self) -> Result { + Err(EAGAIN) + } + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: super::fw::queue::Policy, + ) -> Result { + self.port.qos_publish(owner, scheduler, policy) + } + fn qos_cancel(&mut self, publication: qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn publish_qid(&mut self, _id: kick::Id) -> Result { + // An installed queue published its identity with its first command. + Ok(()) + } + fn publish_outer(&mut self, _priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.port.publish_compute(self.outer.take().ok_or(EIO)?, slot).map(|_| ()) + } + fn notify_compute(&mut self, priority: u8, message: u64) -> Result { + self.port.notify_compute(priority, message) + } + fn compute_written(&self, priority: u8) -> u64 { + self.port.compute_written(priority) + } + fn note_submission(&mut self) -> Result { + Ok(()) } } impl super::Firmware { - fn publish_compute( + #[inline] + fn publish_compute(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if packet.completion.sources().is_some() { + self.publish_compute_inputs(owner, packet, dependencies, deferred) + } else { + self.publish_compute_raw(owner, packet, dependencies, &[], deferred) + } + } + + #[inline(never)] + fn publish_compute_inputs(&mut self, owner: u64, packet: Arc, + dependencies: &[super::fw::kick::KickDependency], deferred: &mut DeferredBatch) -> Result + { + if self.recovery.pending() || super::recovery::Memory::recovery_state(&self.init)? != 0 { + return Err(EAGAIN); + } + let sources = packet.completion.sources().ok_or(EINVAL)?.clone(); + sources.check()?; + let mut merged = super::dependency::Frontier::new(); + for dependency in dependencies { merged.insert(0, *dependency)?; } + let mut holds = KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + let mut native: KVec = + KVec::with_capacity(sources.sources.len(), GFP_KERNEL)?; + for source in &sources.sources { + if source.submission.source_status()? { continue; } + let found = self.queues.compute.iter_mut().flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find_map(|mut queue| match queue.source_dependency(source, packet.completion.status(), + self.recovery.generation()) { + Ok(None) => None, + result => Some(result), + }).ok_or(EAGAIN)??; + let Some((sequence, dependency, stamp_va, event_slot, hold)) = found else { + return Err(EAGAIN); + }; + // The completion-wait record orders the consumer behind its + // source; naming the source in the kick entry as well only adds + // a second, slower wait and a predecessor record. + let wait = super::sync_input::NativeWait { sequence, dependency, stamp_va, event_slot }; + if let Some(old) = native.iter_mut().find(|old| + old.dependency.qid() == dependency.qid()) { + if sequence > old.sequence { *old = wait; } + } else { + native.push(wait, GFP_KERNEL)?; + } + holds.push(hold, GFP_KERNEL)?; + } + let result = self.publish_compute_raw(owner, packet.clone(), merged.as_slice(), + native.as_slice(), deferred); + if packet.is_published() { + for queue in self.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| native.iter().any(|wait| wait.dependency.qid() == queue.qid())) + { + for source in &sources.sources { queue.note_input_consumer(source); } + } + sources.set_holds(holds); + super::sync_input::note_published(); + super::sync_input::note_native(native.len()); + } + result + } + + fn publish_compute_raw( &mut self, owner: u64, packet: Arc, dependencies: &[super::fw::kick::KickDependency], + native: &[super::sync_input::NativeWait], deferred: &mut DeferredBatch, ) -> Result { if self.recovery.pending() { return Err(EAGAIN); } // No descriptor, accounting or inner producer changes precede this - // capacity check. The same device lock covers queue.publish below. - if !self - .init - .work_ready(packet.context.policy().priority(), DataMaster::Compute)? - { - return Err(EAGAIN); - } + // reservation; dropping it unused returns the work-ring slot. + let outer = self.port.reserve_compute(packet.context.policy().priority())?; let queues = &mut self.queues; - let queue = queues + let mut queue = queues .compute .iter_mut() .flatten() @@ -1150,26 +1547,42 @@ impl super::Firmware { } let mut host = ComputeHost { init: &self.init, - primary: &mut self.primary, + port: &self.port, + outer: Some(outer), effort: &mut self.effort, qids: &mut queues.qids, - accounting: &mut queues.accounting, generation: self.recovery.generation(), }; - queue.publish(packet, dependencies, &mut host, &mut |event| { - deferred.push(event) - }) + if native.is_empty() { + queue.publish(packet, dependencies, &mut host, &mut |event| deferred.push(event)) + } else { + queue.publish_with_native_waits(packet, dependencies, native, + &mut host, &mut |event| deferred.push(event)) + } } } impl Registry { + /// Everything ensure_compute establishes for `owner`: the shared compute + /// objects mapped into its VM, and its queue bound to `context` with a + /// pool that fits and no spared witness to release. + fn compute_bound(&self, owner: u64, context: &Arc, required: usize) -> Result { + let Some(queue) = self.compute(owner) else { + return Ok(false); + }; + Ok(!queue.spared_pending() + && queue.pool_fits(required) + && queue.matches(owner, queue.qid(), context)?) + } + fn release_compute_witness( &mut self, init: &super::initdata::InitData, + port: &super::port::Port, owner: u64, deferred: &mut DeferredBatch, ) -> Result { - let Some(queue) = self + let Some(mut queue) = self .compute .iter_mut() .flatten() @@ -1181,10 +1594,9 @@ impl Registry { if !queue.spared_pending() { return Ok(false); } - let view = qos::View::new(init.qos()?)?; queue.release_witnessed( super::recovery::Memory::recovery_state(init)?, - |owner| self.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; Ok(queue.spared_pending()) @@ -1196,7 +1608,7 @@ impl Registry { error: Error, deferred: &mut DeferredBatch, ) -> Result { - for queue in self + for mut queue in self .compute .iter_mut() .flatten() @@ -1223,33 +1635,63 @@ impl super::Firmware { let registry = &mut self.queues; registry.observations.clear(); let mut requested = false; + // Every queue's pending identities are staged, not only the ones this batch + // retires: the scan passes each new record once, and a queue retired by a later + // batch finds its observation already latched. for queue in registry .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) { - requested |= queue.request_observations(&mut registry.observations, masks)?; + requested |= queue.request_observations(&mut registry.observations)?; } + // Completion records are appended in order: fold only the records written since + // the previous scan. A request whose stamp already covers it has its record in the + // ring; if the scan ended before finding one, read the whole ring once. if requested { - self.init - .scan_compute_completions(|record| registry.observations.fold(record))?; + let observations = &mut registry.observations; + let seen = &mut registry.completion_seen; + registry.completion_cursor = self.init.fold_compute_completions( + registry.completion_cursor, + seen, + |record| observations.fold(record), + )?; + if observations.covered_pending() { + registry.completion_cursor = self + .init + .resync_compute_completions(seen, |record| observations.fold_any(record))?; + } } - let view = qos::View::new(self.init.qos()?)?; - for queue in registry + let port = &self.port; + let retired = deferred.pending(); + let mut freed = false; + let mut doorbells = [0u64; 4]; + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.latch_observations(&mut registry.observations, masks)?; + let busy = !queue.channel_free(); + queue.latch_observations(&mut registry.observations)?; queue.retire( masks, generation, state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); + // Withheld doorbells are rung once their command is next but one; one per + // priority class announces every queue's work slots on that ring. + if let Some(doorbell) = queue.due_doorbell(|class| port.compute_announced(class)) { + let class = ((doorbell >> 2) & 3) as usize; + doorbells[class] = doorbell; + } + } + for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { + self.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; } // All compute observations in this batch are latched before a VM // failure can settle its render peers. Exact successes win first. @@ -1257,26 +1699,37 @@ impl super::Firmware { let terminal = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .and_then(compute::Queue::take_terminal); + .and_then(|mut queue| queue.take_terminal()); if let Some((status, error)) = terminal { status.record(error); self.settle_render_vm(&status, error, deferred)?; self.queues.fail_compute_vm(&status, error, deferred)?; } } - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; + freed |= busy && queue.channel_free(); + } + // Wake only the admission waits this batch can end. The fences gate + // scheduler entities alone; their callbacks never take the device mutex. + let shared = &self.primary.state.shared; + if freed { + shared.channels.notify(); + } + if deferred.pending() != retired { + shared.progress.notify(); } Ok(()) } @@ -1286,18 +1739,17 @@ impl Backend { fn ensure_compute(&self, scs_pages: u16, admission: &Admission<'_>) -> Result { admission.check()?; let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; - use kernel::time::{delay::fsleep, Delta}; - const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; - for _ in 0..SPARED_WAIT_MS { + loop { admission.check()?; + let observed = self.shared.progress.observe(); let pending = (|| { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; firmware.replay_compute(&mut deferred)?; firmware .queues - .release_compute_witness(&firmware.init, self.owner, &mut deferred) + .release_compute_witness(&firmware.init, &firmware.port, self.owner, &mut deferred) })(); deferred.finish(); let pending = pending?; @@ -1307,14 +1759,306 @@ impl Backend { result => return result, } } - // Other owners continue submitting while this owner gives its - // queue-local drain witness time to arrive. - fsleep(Delta::from_millis(1)); + // Other owners continue submitting while this owner waits for its + // queue-local drain witness. No new command has been accepted: the + // spared queue and its retained work stay intact past the deadline. + self.shared.queue_events(); + self.wait_release(&self.shared.progress, observed, admission)?; + } + } + + /// Steady-state publication into this owner's bound channel without the device mutex: only + /// the channel's own lock and the port's small locks are taken, and nothing device-wide is + /// scanned. None leaves the packet untouched for the device-mutex path, which owns binding, + /// first installation, input sources, render prefixes, replay and recovery. + fn compute_direct( + &self, + packet: &Arc, + required: usize, + deferred: &mut DeferredBatch, + ) -> Option { + if packet.completion.sources().is_some() + || self.context.render_seen() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let cell = self.direct.lock().clone()?; + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire() { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let mut queue = cell.lock()?; + if !queue.installed() + || queue.spared_pending() + || !queue.pool_fits(required) + || queue.pending_switch_from().is_some() + || !matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)) + || !lease.is_current() + { + return None; + } + let outer = port.reserve_compute(packet.context.policy().priority()).ok()?; + let mut host = DirectHost { + port, + outer: Some(outer), + generation: port.generation(), + }; + match queue.publish(packet.clone(), &[], &mut host, &mut |event| deferred.push(event)) { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => Some(result), } - // No new command has been accepted. Keep the spared queue and its - // retained work intact until a later admission observes its witness. - admission.check()?; - Err(EBUSY) + } + + /// Scheduler admission of a compute packet. Shared capacity the packet lacks (a channel, a + /// work state, retirement-bound room, reopened publication) becomes a fence that the release + /// providing it signals, so publication finds it ready instead of sleeping for it. Errors + /// and anything prepare cannot judge are left to publication. + pub(super) fn prepare_compute(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return None; }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()).ok()?; + loop { + let channels = self.shared.channels.observe(); + let progress = self.shared.progress.observe(); + let (domain, observed) = match self.compute_readiness(packet, required) { + Readiness::Ready => return None, + Readiness::Channel => (&self.shared.channels, channels), + Readiness::Progress => (&self.shared.progress, progress), + }; + self.shared.queue_events(); + match domain.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + + fn compute_readiness(&self, packet: &Arc, required: usize) -> Readiness { + match packet.completion.work_state().and_then(|work| work.acquire()) { + Err(error) if error == EBUSY => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(_) => {} + } + // The steady state: this owner's channel is bound and installed. + if packet.completion.sources().is_none() { + if let Some(cell) = self.direct.lock().clone() { + let bound = cell.lock().is_some_and(|mut queue| { + let bound = queue.installed() + && !queue.spared_pending() + && queue.pool_fits(required) + && matches!(queue.matches(self.owner, queue.qid(), &self.context), Ok(true)); + if bound { + queue.claim(); + } + bound + }); + if bound { + self.hold_claim(Some(cell)); + return Readiness::Ready; + } + } + } + match self.sources_published(packet) { + Ok(false) => return Readiness::Progress, + Err(_) => return Readiness::Ready, + Ok(true) => {} + } + let lease = match self.shared.preparations.try_enter() { + Ok(lease) => lease, + Err(error) if error == EAGAIN => return Readiness::Progress, + Err(_) => return Readiness::Ready, + }; + let state = self.shared.state.lock(); + let Some(firmware) = state.as_deref() else { return Readiness::Ready; }; + if !lease.is_current() + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init).unwrap_or(0) != 0 + { + return Readiness::Progress; + } + firmware.queues.compute_readiness(self.owner, Some(&self.context), required) + } + + /// Replaces the prepared packet's channel claim; None releases it after publication, + /// cancellation or close. + pub(super) fn hold_claim(&self, cell: Option>) { + let old = core::mem::replace(&mut *self.claimed.lock(), cell); + let freed = old.as_ref().and_then(|old| old.lock()).is_some_and(|mut queue| { + let busy = !queue.channel_free(); + queue.release_claim(); + busy && queue.channel_free() + }); + if freed { + // A channel nobody prepares into any more is shareable now. + self.shared.channels.notify(); + } + } + + /// Called with the device mutex held after a successful publication: remembers the bound + /// channel for direct publication. + fn remember_direct(&self, registry: &Registry) { + let cell = registry.compute_cell(self.owner); + let mut direct = self.direct.lock(); + let current = match (&*direct, &cell) { + (Some(old), Some(new)) => Arc::ptr_eq(old, new), + (None, None) => true, + _ => false, + }; + if !current { + let old = core::mem::replace(&mut *direct, cell); + drop(direct); + drop(old); + } + } + + /// Publishes a dependency-free compute command from its submit ioctl when + /// its queue is already bound and nothing has to wait, as renders do. Any + /// wait leaves the command, unpublished, to its scheduler. + pub(super) fn try_compute_early(&self, packet: &Arc) -> bool { + if self.compute_backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = match self.compute_direct_packet(packet, &mut deferred) { + Some(result) => result, + None => self.compute_inline(packet, &mut deferred), + }; + if matches!(result, Err(error) if error == render::FALLBACK) && !packet.is_published() { + deferred.finish(); + packet.abandon_publication(); + return false; + } + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } + + fn compute_direct_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Option { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { + return Some(Err(EINVAL)); + }; + let required = match super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages()) { + Ok(required) => required, + Err(error) => return Some(Err(error)), + }; + self.compute_direct(packet, required, deferred) + } + + /// compute_packet's publication section without its waits: every condition + /// it would sleep or retry for returns FALLBACK instead. + fn compute_inline( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if packet.cancel_requested() || self.admission_open().is_err() || packet.completion.status().get() != 0 { + return Err(render::FALLBACK); + } + let lease = self.enter_preparation(false)?; + match packet.completion.work_state()?.acquire() { + Err(error) if error == EBUSY => return Err(render::FALLBACK), + result => { + result?; + } + } + // A lone admission keeps the original latency-sensitive probe. + // Competing publishers otherwise spin on a binding they do not own. + if !self.context.compute_binding_hint() + && self.shared.compute_demand.load(Ordering::Relaxed) > 1 + { + return Err(render::FALLBACK); + } + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() + || self.shared.crashed.load(Ordering::Acquire) + || firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(render::FALLBACK); + } + firmware.replay_compute(deferred)?; + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + return Err(render::FALLBACK); + } + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + match firmware.publish_compute(self.owner, packet.clone(), dependencies.as_slice(), deferred) { + Err(error) if (error == EBUSY || error == EAGAIN) && !packet.is_published() => { + Err(render::FALLBACK) + } + result => { + if result.is_ok() { + self.remember_direct(&firmware.queues); + } + result + } + } + } + + /// A scheduled dependency may refer to a previous recovery epoch. Do not + /// acquire consumer capacity until every live source is published now. + fn sources_published(&self, packet: &super::job::Packet) -> Result { + let Some(sources) = packet.completion.sources() else { return Ok(true); }; + sources.check()?; + let state = self.shared.state.lock(); + let firmware = state.as_deref().ok_or(ENODEV)?; + if firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0 + { return Ok(false); } + for source in &sources.sources { + if source.submission.source_status()? { continue; } + if !firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&source.submission, + packet.completion.status(), firmware.recovery.generation())) + { return Ok(false); } + } + Ok(true) } /// Called after taking this packet's publication claim. All waits and work @@ -1326,8 +2070,17 @@ impl Backend { ) -> Result { packet.check_dependencies()?; let super::command::Validated::Compute { scratch, .. } = &packet.command else { return Err(EINVAL); }; - let mut recovery_polls = 0; - let mut state_delay = 20i64; + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scratch.pages())?; + if let Some(result) = self.compute_direct(packet, required, deferred) { + return result; + } + // A bound queue needs none of ensure_compute's admission passes. Try + // without them; the publication section below checks the same state. + // Every retry blocks on the release that ends its wait: retirement + // progress (work states, ring room, replays) and the reopening of + // publication after recovery both signal `progress`. + let mut bound = true; + let mut demand = None; let deadline = now_ns().saturating_add(WORK_RING_WAIT_NS); let check = || { if packet.cancel_requested() { return Err(ECANCELED); } @@ -1336,24 +2089,30 @@ impl Backend { Ok(()) }; let admission = Admission::new(deadline, &check); + let progress = &self.shared.progress; loop { admission.check()?; + let observed = progress.observe(); + if !self.sources_published(packet)? { + self.shared.queue_events(); + self.wait_release(progress, observed, &admission)?; + continue; + } // All DRM dependencies were satisfied before this scheduler callback. // Recheck on every retry: pressure may yield an idle binding between // admission and publication without changing this logical queue. - Backend::ensure_compute(self, scratch.pages(), &admission)?; - admission.check()?; + if !bound { + demand.get_or_insert_with(|| ComputeDemand::new(&self.shared)); + Backend::ensure_compute(self, scratch.pages(), &admission)?; + admission.check()?; + } let lease = self.shared.preparations.enter_until(&admission)?; // Work states free as this context's earlier commands retire. - // Wait for one under the packet's admission deadline, as render - // publication does, instead of failing the packet. match packet.completion.work_state()?.acquire() { Err(error) if error == EBUSY => { drop(lease); - admission.check()?; self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); + self.wait_release(progress, observed, &admission)?; continue; } result => { @@ -1376,18 +2135,17 @@ impl Backend { } let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; if firmware.recovery.pending() || recovery_state != 0 { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } firmware.replay_compute(deferred)?; - if firmware.queues.compute(self.owner).is_none() { + if !firmware.init.compute_vm_cached(self.context.vm()) + || !firmware.queues.compute_bound(self.owner, &self.context, required)? + { + bound = false; drop(state); drop(lease); continue; @@ -1400,29 +2158,22 @@ impl Backend { dependencies.as_slice(), deferred, ); - if matches!(result, Err(EBUSY) | Err(EAGAIN)) - && !packet.is_published() - && (firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0) + if !packet.is_published() + && (matches!(result, Err(EAGAIN)) + || (matches!(result, Err(EBUSY)) + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0))) { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; + // Recovery or backpressure: nothing is firmware-visible. Other + // queues keep submitting and the event worker retires meanwhile. drop(state); drop(lease); self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + self.wait_release(progress, observed, &admission)?; continue; } - if matches!(result, Err(EAGAIN)) && !packet.is_published() { - // Backpressure owns no firmware-visible attempt. Other queues - // can submit and the event worker can retire while we sleep. - drop(state); - drop(lease); - self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; + if result.is_ok() { + self.remember_direct(&firmware.queues); } return result; } @@ -1435,7 +2186,7 @@ impl super::Firmware { /// or an allocation-dependent collector. fn finish_stopped_queues(&mut self) { self.stop_render(); - for queue in self + for mut queue in self .queues .compute .iter_mut() @@ -1491,6 +2242,7 @@ impl super::Shared { dev_warn!(self.dev.as_ref(), "Stopped GPU retains outstanding preparation owners\n"); return; } + self.forget_port(); let firmware = self.state.lock().take(); if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); } } @@ -1504,6 +2256,7 @@ impl super::Shared { pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); let _stopping = self.stop.lock(); + self.forget_port(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { @@ -1543,20 +2296,41 @@ impl super::Firmware { deferred: &mut DeferredBatch, now: u64, ) -> Result> { - for queue in self + let mut freed = false; + let mut releasing = false; + for mut queue in self .queues .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + let busy = !queue.channel_free(); queue.recycle_killed()?; queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } + freed |= busy && queue.channel_free(); + releasing |= queue.pool_release_pending(); + } + freed |= self.queues.observe_pool_releases(&self.init, deferred)? != 0; + let shared = &self.primary.state.shared; + // Edge notifications cover the transitions this driver makes; every + // event pass also rechecks the level, so capacity that appeared through + // another engine (render QIDs, closed logical queues, render-held work + // states) still ends a scheduler wait that has no timeout of its own. + if freed + || (shared.channels.has_waiters() + && self.queues.compute_readiness(0, None, 0) == Readiness::Ready) + { + shared.channels.notify(); + } + // A pool release has no completion event: each observation pass is + // the progress a growing owner waits for. + if releasing || shared.progress.has_waiters() { + shared.progress.notify(); } - self.queues.observe_pool_releases(&self.init, deferred)?; if self.queues.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) .any(|queue| queue.pool_release_stalled(now)) { return Err(ETIMEDOUT); @@ -1568,6 +2342,11 @@ impl super::Firmware { // retained compute pool stays warm: render and new compute need it more. let cap = if super::freelist::lower_pressure() { 0 + } else if self.primary.state.shared.compute_demand.load(Ordering::Acquire) != 0 { + // A retired channel being requested by another runnable publisher + // is not an unused warm pool. Keep its existing backing through + // handoff; real allocation pressure above still forces reclamation. + super::kick::COMPUTE_CHANNELS } else { COMPUTE_WARM_CAP }; @@ -1598,8 +2377,9 @@ impl Registry { self.compute.iter().flatten().count(), self.compute.len(), self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()).filter(|queue| queue.reusable()).count(), self.compute.iter().flatten().filter(|entry| entry.queue.is_none()).count())?; - writeln!(f, "compute_channels used={} total={}", - self.compute.iter().flatten().count(), super::kick::COMPUTE_CHANNELS)?; + writeln!(f, "compute_channels used={} fresh_fits={}", + self.compute.iter().flatten().count(), + u8::from(self.qids.compute_fits(self.admission.queues() as usize)))?; writeln!(f, "scheduler_teardown pending={}", self.teardown.len())?; for queue in self.compute.iter().flatten().filter_map(|entry| entry.queue.as_deref()) { queue.write_capacity(f)?; @@ -1609,13 +2389,42 @@ impl Registry { } impl Backend { - pub(super) fn wait_capacity(&self, started: u64) -> Result { - let check = || self.admission_open(); - let admission = Admission::new(started.saturating_add(WORK_RING_WAIT_NS), &check); - self.wait_compute_capacity(&admission) + /// Compute admission without a channel: give back what pressure allows, then block until a + /// channel is released after `observed` (sampled before the failed check). + fn wait_channels(&self, observed: u64, admission: &Admission<'_>, reclaim_backing: bool) -> Result { + self.reclaim_capacity(admission, reclaim_backing)?; + self.wait_release(&self.shared.channels, observed, admission) + } + + /// Blocks until `domain` is signalled after `observed`, this packet's admission ends, or one + /// admission slice passes: cancellation and logical close have no release to wait for. + fn wait_release( + &self, + domain: &super::availability::Availability, + observed: u64, + admission: &Admission<'_>, + ) -> Result { + use kernel::dma_fence::RawDmaFence; + let slice = admission.slice_ms()?; + if let Some(fence) = domain.wait(observed)? { + // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait. + let waited = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(slice) as isize, + ) + }; + if waited < 0 { + return Err(Error::from_errno(waited as i32)); + } + } + admission.check() } - fn wait_compute_capacity(&self, admission: &Admission<'_>) -> Result { + /// Reclaim steps of a capacity wait that never sleep: observe consumed pool releases and, + /// under backing pressure, give idle pools back. + fn reclaim_capacity(&self, admission: &Admission<'_>, reclaim_backing: bool) -> Result { admission.check()?; let mut deferred = DeferredBatch::publication()?; let result: Result = (|| { @@ -1630,14 +2439,15 @@ impl Backend { if firmware.recovery.pending() || super::recovery::Memory::recovery_state(&firmware.init)? != 0 { return Ok(()); } - for queue in firmware.queues.compute.iter_mut().flatten().filter_map(|entry| entry.queue.as_deref_mut()) { - admission.check()?; - queue.share_idle()?; - } // Consumption, not publication or an elapsed timeout, frees backing. firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; - firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; - if !firmware.render_control_backpressured() { + // Pool eviction addresses backing allocation pressure, not channel + // occupancy. Normal idle expiry and the retained-pool cap still run + // in the event worker for either kind of admission wait. + if reclaim_backing { + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + } + if reclaim_backing && !firmware.render_control_backpressured() { let (published, _) = firmware.queues.vacate_idle_pools(&firmware.init, 0, now_ns(), 0)?; if published != 0 { firmware.primary.notify(super::MSG_CONTROL_NOTIFY)?; } } @@ -1649,8 +2459,6 @@ impl Backend { Err(error) if error == EAGAIN => {} result => result?, } - admission.check()?; - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); admission.check() } } @@ -1669,12 +2477,17 @@ mod shared_capacity_tests { ) .unwrap(); let mut leases = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); - // Physical compute channels are device-owned and bounded independently - // of how many logical clients have queued unsignalled work. - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + let mut slots = KVec::with_capacity(crate::mmu::MAX_EXECUTION_CONTEXTS as usize, GFP_KERNEL).unwrap(); + for _ in 0..crate::mmu::MAX_EXECUTION_CONTEXTS { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // With every execution context open, compute keeps only the channels + // that leave each logical queue a dedicated render pair. + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert!(matches!(registry.reserve_compute(1000), Err(EAGAIN))); + assert!(registry.qids.render_fits(slots.len())); for owner in 1..=u64::from(crate::mmu::MAX_EXECUTION_CONTEXTS) { let lease = registry.admission.acquire_render(owner, None, u64::MAX, || Ok(())).unwrap(); registry.ensure_render_ids(lease.slot(), owner).unwrap(); @@ -1695,9 +2508,6 @@ mod shared_capacity_tests { GFP_KERNEL, ) .unwrap(); - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { - registry.reserve_compute(owner).unwrap(); - } let mut slots = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); let mut leases = KVec::with_capacity(limit as usize, GFP_KERNEL).unwrap(); for owner in 1..=u64::from(limit) { @@ -1726,11 +2536,31 @@ mod shared_capacity_tests { leases.push(lease, GFP_KERNEL).unwrap(); } } - for owner in 1..=super::super::kick::COMPUTE_CHANNELS as u64 { + for owner in 1..=super::super::kick::COMPUTE_FLOOR as u64 { registry.reserve_compute(owner).unwrap(); } assert_eq!(leases.len(), 36); } + + #[test] + fn compute_grows_with_free_qids_and_narrows_queue_admission() { + let mut registry = KBox::init(Registry::new(1, Deflake::ONE_CLUSTER, [0; 2], &V26A428), GFP_KERNEL).unwrap(); + let mut slots = KVec::with_capacity(20, GFP_KERNEL).unwrap(); + for _ in 0..20 { + slots.push(registry.admission.reserve_queue().unwrap(), GFP_KERNEL).unwrap(); + } + // Sixteen open compute clients each keep their own channel. + for owner in 1..=16u64 { + registry.reserve_compute(owner).unwrap(); + } + // Growth stops where open queues plus headroom would lose their pairs. + let mut owner = 17; + while registry.reserve_compute(owner).is_ok() { + owner += 1; + } + assert!(registry.qids.render_fits(slots.len() + 24)); + assert!(!registry.qids.render_fits(crate::mmu::MAX_EXECUTION_CONTEXTS as usize)); + } } impl Backend { diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7d690470cd7b5..d19f51c08dd29 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -13,7 +13,14 @@ use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; impl Backend { pub(super) fn release_backlog(&self, packet: &Packet) { if packet.take_backlog() { - self.backlog.fetch_sub(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_sub(1, Ordering::AcqRel); + } + } + + pub(super) fn backlog_of(&self, packet: &Packet) -> &core::sync::atomic::AtomicU32 { + match packet.engine() { + Engine::Render => &self.backlog, + Engine::Compute => &self.compute_backlog, } } @@ -48,6 +55,7 @@ impl Backend { drop(state); drop(binding); self.finish_uninstalled_compute(retired)?; + self.shared.channels.notify(); self.shared.queue_events(); return Ok(()); } @@ -85,6 +93,7 @@ impl Backend { // A builder may own only an off-lock reservation when close arrives. // Record intent before scanning queues; every later install/rearm checks it. self.closed.store(true, Ordering::Release); + self.hold_claim(None); let mut state = self.shared.state.lock(); let Some(firmware) = (*state).as_deref_mut() else { return Ok(()); @@ -99,12 +108,63 @@ impl Backend { firmware.queues.release_render(self.owner)?; firmware.queues.admission.release_owner(self.owner); drop(state); + // Its idle channel, or its slot and never-published QID, is free. + self.shared.channels.notify(); self.shared.queue_events(); Ok(()) } } impl job::Backend for Backend { + fn input_sources(&self, inputs: &[crate::file::SyncItem]) + -> Option> + { + use crate::g17::{fence, sync_input}; + let mut heads = KVec::new(); + let mut sources: KVec = KVec::new(); + let mut publication_waits = KVec::new(); + for input in inputs.iter().filter_map(|input| input.fence.as_ref()) { + let status = job::fence_status(input); + if status > 0 { continue; } + if status < 0 { return None; } + let leaves = sync_input::collect(input)?; + for leaf in &leaves { + if leaf.status > 0 { continue; } + if leaf.status < 0 { return None; } + // SAFETY: Collection owns each leaf until this lookup finishes. + let submission = unsafe { fence::exported_source_raw(leaf.fence) }?; + if submission.source_status().ok()? { continue; } + if !submission.same_vm(self.context.status()) { return None; } + if sources.iter().any(|known| Arc::ptr_eq(&known.submission, &submission)) { + continue; + } + if sources.len() == crate::g17::fw::kick::KICK_BARRIERS_MAX { return None; } + let published = { + let state = self.shared.state.lock(); + let firmware = state.as_deref()?; + if self.shared.crashed.load(Ordering::Acquire) || firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init).ok()? != 0 + { return None; } + firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.has_input_source(&submission, self.context.status(), + firmware.recovery.generation())) + }; + if !published { + if submission.source_status().ok()? { continue; } + // The existing scheduler fence becomes ready after actual + // publication, not GPU completion. No binding loan or + // consumer hardware capacity is acquired while waiting. + publication_waits.push(submission.ready()?, GFP_KERNEL).ok()?; + } + sources.push(sync_input::Source { submission }, GFP_KERNEL).ok()?; + } + heads.push(input.clone(), GFP_KERNEL).ok()?; + } + if sources.is_empty() { return None; } + sync_input::Sources::new(heads, sources, publication_waits).ok() + } + fn feed(&self) -> Option> { Some(self.shared.feed.clone()) } @@ -156,8 +216,13 @@ impl job::Backend for Backend { /// Firmware kick parents order the same engine. Explicit cross-engine and /// input dependencies were already attached to the DRM scheduled job. - fn prepare(&self, _packet: &Packet) -> Option { - None + /// Compute and render admission wait for their shared capacity here, as a + /// fence. + fn prepare(&self, packet: &Arc) -> Option { + match packet.engine() { + Engine::Compute => self.prepare_compute(packet), + Engine::Render => self.admit_render(packet), + } } fn publish(&self, packet: Arc) -> Result { @@ -185,6 +250,9 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -213,6 +281,9 @@ impl job::Backend for Backend { fn cancel(&self, packet: Arc) { self.release_backlog(&packet); + if packet.engine() == Engine::Compute { + self.hold_claim(None); + } if packet.request_cancel() { Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); return; @@ -274,11 +345,14 @@ impl job::Backend for Backend { } fn try_early(&self, packet: &Arc) -> bool { - self.try_render_early(packet) + match packet.engine() { + Engine::Render => self.try_render_early(packet), + Engine::Compute => self.try_compute_early(packet), + } } fn count_backlog(&self, packet: &Packet) { packet.count_backlog(); - self.backlog.fetch_add(1, Ordering::AcqRel); + self.backlog_of(packet).fetch_add(1, Ordering::AcqRel); } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index c407d9576e9c4..42282aea62023 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -20,11 +20,10 @@ impl Backend { return Ok(Decision::Own); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute(self.owner) - .filter(|queue| queue.can_classify_timeout(packet)) else { - return Ok(Decision::Own); - }; - let Some(own) = queue.timeout_head() else { + // Each queue is locked on its own: release this one before visiting the others. + let Some(own) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) + .and_then(|queue| queue.timeout_head()) else { return Ok(Decision::Own); }; let computes = firmware.queues.compute.iter().flatten() @@ -64,13 +63,20 @@ impl Backend { return Ok(None); }; firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute_mut(self.owner) else { + let Some(mut queue) = firmware.queues.compute_mut(self.owner) else { return Ok(None); }; if !queue.owns_packet(packet) { return Ok(None); } - queue.timeout(&mut |event| deferred.push(event)) + let status = queue.timeout(&mut |event| deferred.push(event)); + // A timed-out queue leaves ordinary retirement: announce what it withheld. + let withheld = queue.take_withheld_doorbell(); + drop(queue); + if let Some(doorbell) = withheld { + firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; + } + status })(); if result.as_ref().is_ok_and(|status| status.is_some()) { // VM status and peer settlement precede signalling this timeout. The backend @@ -91,8 +97,13 @@ impl Backend { return Ok(()); }; firmware.scan_compute(None, &mut deferred)?; - if let Some(queue) = firmware.queues.compute_mut(self.owner) { + if let Some(mut queue) = firmware.queues.compute_mut(self.owner) { queue.cancel(packet, &mut |event| deferred.push(event))?; + let withheld = queue.take_withheld_doorbell(); + drop(queue); + if let Some(doorbell) = withheld { + firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell)?; + } } Ok(()) })(); diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 984b6c1f643eb..bf9d023038a51 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -9,6 +9,7 @@ use crate::g17::{ fw::channels::{ContextKill, ControlRecord}, queue::compute::exit, recovery::Memory as _, + timeout::LIVENESS_NS, Firmware, MSG_CONTROL_NOTIFY, }; use core::sync::atomic::{fence, Ordering}; @@ -19,7 +20,7 @@ impl super::Registry { /// batch cannot overflow a separate cache. No allocation or wait is needed. pub(super) fn acknowledge_compute_exit(&mut self, cookie: u64) { for entry in self.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { if queue.acknowledge_exit(cookie) { return; } @@ -31,12 +32,12 @@ impl super::Registry { impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { let recovery_closed = self.init.recovery_state()? == 0; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.settle_exit( recovery_closed, - &mut |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -48,7 +49,7 @@ impl Firmware { // Publication has already retained each kill. Retry only the doorbell; // failed attempts do not extend its bounded polling window. for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { let _ = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)); } } @@ -82,17 +83,19 @@ impl Firmware { }) else { return Ok(()); }; - if self.render_control_backpressured() { + if self.render_control_backpressured() || self.kill_unsafe(context, now) { return Ok(()); } - let queue = self.queues.compute[index] - .as_ref() - .and_then(|entry| entry.queue.as_deref()) - .ok_or(EIO)?; - let Some(owner) = queue.exit_owner(context) else { - return Ok(()); + let (owner, target) = { + let queue = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .ok_or(EIO)?; + let Some(owner) = queue.exit_owner(context) else { + return Ok(()); + }; + (owner, queue.qid()) }; - let target = queue.qid(); let (mut own, mut foreign, slots) = self.queues.render.exit_scope(context); for entry in self.queues.compute.iter().flatten() { if let Some(queue) = entry.queue.as_deref() { @@ -130,7 +133,7 @@ impl Firmware { } // Publication is irrevocable even if its doorbell fails. Keep both // the killed graph and the held render identities until processor stop. - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; @@ -147,6 +150,29 @@ impl Firmware { Ok(()) } + /// A kill while other contexts' commands run makes the firmware restart the + /// GPU, and repeated ones crash it; on an otherwise idle GPU it is safe. A + /// closed context's commands left alone retire like any others once the GPU + /// reaches them, so its queues stay quarantined and retained until then. Only + /// a GPU that retired nothing for [`LIVENESS_NS`] has stopped, and then the + /// kill proceeds. + fn kill_unsafe(&self, context: &Arc, now: u64) -> bool { + let stalled = crate::g17::recovery::since_retired(now).is_none_or(|since| since >= LIVENESS_NS); + if stalled { + return false; + } + self.queues.render.foreign_in_flight(context) + || self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| { + queue.in_flight() && !queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + }) + } + pub(in crate::g17) fn compute_exit_polling(&self, now: u64) -> bool { if self .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index d71f410c76fb4..32b9acef94c0f 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -3,7 +3,7 @@ //! Ordered republication of witnessed, unexecuted compute commands. use super::DeferredBatch; -use crate::g17::{completion::Deferred, qos, recovery, Firmware}; +use crate::g17::{completion::Deferred, recovery, Firmware}; use kernel::prelude::*; impl Firmware { @@ -31,12 +31,12 @@ impl Firmware { return Ok(()); } deferred.reserve_replays(commands)?; - let view = qos::View::new(self.init.qos()?)?; + let port = &self.port; for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { + if let Some(mut queue) = entry.queue.as_deref_mut() { queue.prepare_replays( state, - |owner| self.queues.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -46,20 +46,14 @@ impl Firmware { if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { return Ok(()); } - let Some(queue) = self.queues.compute[index] + let Some((packet, owner)) = self.queues.compute[index] .as_ref() .and_then(|entry| entry.queue.as_deref()) + .and_then(|queue| Some((queue.replay_front()?, queue.owner()))) else { break; }; - let Some(packet) = queue.replay_front() else { - break; - }; - let owner = queue.owner(); - let outer_ready = self.init.work_ready( - packet.context.policy().priority(), - crate::g17::fw::queue::DataMaster::Compute, - )?; + let outer_ready = self.port.compute_ready(packet.context.policy().priority())?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); @@ -90,7 +84,7 @@ impl Firmware { let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); - let queue = self.queues.compute[index] + let mut queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) .ok_or(EIO)?; @@ -110,6 +104,8 @@ impl Firmware { } } self.queues.replay_outer_wait = None; + // Publication behind these replays may proceed. + self.primary.state.shared.progress.notify(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4fd6e56741986..4cf2dd0fafb97 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -131,8 +131,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entries(&self) -> Result { - self.firmware.init.entries() + fn entry(&self, index: usize) -> Result<(u32, u32)> { + self.firmware.init.entry(index) } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -203,6 +203,22 @@ impl recovery::Queues for Service<'_> { } self.firmware.queues.classify_render(class) } + fn note_interrupted(&mut self, qid: u8, stamp: u32) -> recovery::Interrupted { + let render = self.firmware.queues.note_render_interrupted(qid, stamp); + if render != recovery::Interrupted::None { + return render; + } + self.firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.qid() == qid) + .map_or(recovery::Interrupted::None, |queue| { + queue.note_interrupted(stamp) + }) + } fn settle_blamed_render(&mut self, qid: u8, error: Error) { let result = self .firmware @@ -220,14 +236,34 @@ impl recovery::Queues for Service<'_> { let result = self.firmware.release_compute_witnesses(self.deferred); self.settlement_error = self.settlement_error.or(result.err()); } + fn announce_withheld(&mut self) { + let mut doorbells = [0u64; 4]; + for mut queue in self + .firmware + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if let Some(doorbell) = queue.take_withheld_doorbell() { + doorbells[((doorbell >> 2) & 3) as usize] = doorbell; + } + } + // One doorbell per priority class announces every queue's work slots on that ring. + for doorbell in doorbells.into_iter().filter(|doorbell| *doorbell != 0) { + let result = self.firmware.port.notify_compute(((doorbell >> 2) & 3) as u8, doorbell); + self.settlement_error = self.settlement_error.or(result.err()); + } + } } impl Firmware { fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { let state = recovery::Memory::recovery_state(&self.init)?; - let view = crate::g17::qos::View::new(self.init.qos()?)?; + let port = &self.port; let registry = &mut self.queues; - for queue in registry + for mut queue in registry .compute .iter_mut() .flatten() @@ -235,7 +271,7 @@ impl Firmware { { queue.release_witnessed( state, - |owner| registry.accounting.complete(&view, &[owner]), + |owner| port.qos_complete(&[owner]), &mut |event| deferred.push(event), )?; } @@ -266,7 +302,10 @@ impl Firmware { /// Preparation admission is already closed and all builders have joined. pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { - if self.recovery.restart_limit(super::now_ns()) { return Err(ETIMEDOUT); } + if self.recovery.restart_limit(super::now_ns()) { + pr_warn!("G17: restart storm limit reached\n"); + return Err(ETIMEDOUT); + } let shared = self.primary.state.shared.clone(); let mut service = Service { firmware: self, @@ -276,7 +315,11 @@ impl Firmware { }; let result = recovery::service(&mut service, || { shared.crashed.load(core::sync::atomic::Ordering::Acquire) - })?; + }); + // Publishers without the device mutex stamp their tickets with this + // mirror; they rejoin only after preparation admission reopens. + service.firmware.port.set_generation(service.firmware.recovery.generation()); + let result = result?; if let Some(error) = result.target_error { dev_warn!( shared.dev.as_ref(), @@ -319,8 +362,10 @@ impl crate::g17::Shared { }; deferred }; + crate::g17::trace::event(0); let result = self.runtime_event_turn(&mut deferred); deferred.finish(); + crate::g17::trace::event(2); self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); @@ -348,6 +393,7 @@ impl crate::g17::Shared { (recover, masks) }; deferred.finish(); + crate::g17::trace::event(1); if recover { self.preparations.wait_drained_until(super::now_ns().saturating_add(super::WORK_RING_WAIT_NS))?; } @@ -392,6 +438,11 @@ impl crate::g17::Shared { .any(|queue| queue.in_flight()); let in_flight = compute || firmware.queues.render_in_flight(); let idle = firmware.idle_check(now, in_flight)?; + let lingers = if firmware.recovery.pending() { + None + } else { + firmware.expire_render_lingers(now)? + }; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; let pools = firmware.service_compute_pools(deferred, now)?; @@ -434,11 +485,14 @@ impl crate::g17::Shared { } else { idle }; - // A retained compute pool gives its backing back when it expires. - let idle = match (idle, pools) { - (Some(delay), Some(expiry)) => Some(delay.min(expiry)), - (delay, expiry) => delay.or(expiry), - }; + // A retained compute pool gives its backing back when it expires, and + // so does an idle render free-list registration. + let idle = [pools, lingers] + .into_iter() + .fold(idle, |idle, expiry| match (idle, expiry) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }); if let Some(delay) = idle { self.queue_idle(delay); } @@ -460,6 +514,11 @@ impl crate::g17::Shared { if grow_ready { self.queue_grow(); } + // Firmware ring room, consumed releases and teardown steps have no + // event of their own: recheck render publishers' conditions every pass. + if self.renders.has_waiters() { + self.renders.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 59db76b9f109a..b5649f059f5d2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -6,6 +6,7 @@ mod exit; mod dependencies; +mod direct; mod grow; mod observe; mod pools; @@ -16,6 +17,7 @@ mod settle; mod submit; pub(super) mod teardown; +pub(super) use submit::FALLBACK; use super::{DeferredBatch, Registry}; use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; use crate::hw::t8140::queues::RENDER_SLOTS; @@ -325,6 +327,28 @@ impl Registry { Ok(None) } + /// The installed pair in `slot` belongs to `owner` and its context, is + /// neither borrowed nor settling a failure, and its pool has backing: the + /// state in which building the pair would do nothing. + pub(super) fn render_ready(&self, slot: u8, owner: u64, context: &Arc) -> Result { + let Ok(entry) = self.render.entry(slot) else { + return Ok(false); + }; + if entry.pending_error.is_some() + || entry.pending_timeout + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + { + return Ok(false); + } + let Some(pair) = entry.pair.as_ref() else { + return Ok(false); + }; + Ok(pair.matches(owner, context)? && !pair.pool().vacant()) + } + fn has_render_ids(&self, owner: u64) -> bool { self.render .entries @@ -507,27 +531,16 @@ impl crate::g17::queue::render::Host for Host<'_> { core::sync::atomic::fence(Ordering::Acquire); Ok(stamps) } - fn qos_publish( + fn qos_publish_pair( &mut self, - owner: crate::g17::qos::Owner, + owners: [crate::g17::qos::Owner; 2], scheduler: u64, policy: crate::g17::fw::queue::Policy, - ) -> Result { - use crate::g17::qos; - self.firmware.queues.accounting.publish( - &qos::View::new(self.firmware.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.firmware.port.qos_publish_pair(owners, scheduler, policy) } fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { - use crate::g17::qos; - self.firmware - .queues - .accounting - .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) + self.firmware.port.qos_cancel(publication) } /// Idle descent is held for the whole first install. A point-in-time power /// check let the firmware gate the cores between the host's configuration @@ -558,7 +571,7 @@ impl crate::g17::queue::render::Host for Host<'_> { self.firmware.queues.qids.publish(id) } fn next_publication(&mut self) -> Result { - self.firmware.queues.qids.next_publication() + Ok(self.firmware.port.next_publication()) } fn publish_outer_pair( &mut self, @@ -599,6 +612,9 @@ impl crate::g17::queue::render::Host for Host<'_> { /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); + crate::g17::trace::doorbell(crate::g17::trace::RENDER, || { + self.firmware.bridge.regs.g17_core_power() + }); let result = self .firmware .primary @@ -618,13 +634,20 @@ impl crate::g17::queue::render::Host for Host<'_> { generation: u64, buffer_id: u8, ) -> Result { - use crate::g17::qos; let queues = &mut self.firmware.queues; - let next = queues.render.pools.completed(pool_id, generation)?; + // Each final release is a control record the firmware processes before + // it returns to the work rings. While other pairs keep it busy, a + // registration left idle lingers instead, and the owner's next render + // reuses it. On an otherwise idle device the release is cheaper than + // reusing a registration that went idle, so it is sent at once. + // The pair retiring here is lent out of its entry meanwhile. + let linger = queues.render.entries.iter().flatten().any(|entry| { + entry.pair.as_ref().is_some_and(|pair| pair.published_in_flight()) + }); + let next = queues.render.pools.completed(pool_id, generation, linger)?; queues.buffers.can_release(buffer_id)?; - let view = qos::View::new(self.firmware.init.qos()?)?; if let Some(owners) = owners { - queues.accounting.complete(&view, &owners)?; + self.firmware.port.qos_complete(&owners)?; } queues.render.pools.commit_completed(pool_id, next); queues.buffers.release(buffer_id) @@ -657,6 +680,30 @@ impl crate::g17::Firmware { result } + /// Event-worker turn: ends free-list lingers that elapsed or whose owner + /// closed or moved on, publishes their releases, and returns the delay to + /// the next expiry, which the worker turns into a timed wake. + pub(in crate::g17) fn expire_render_lingers(&mut self, now: u64) -> Result> { + let render = &mut self.queues.render; + let entries = &render.entries; + let (ended, next) = render.pools.expire_lingers(now, |owner| { + entries.iter().flatten().any(|entry| { + !entry.closed + && entry + .pool + .as_ref() + .is_some_and(|pool| pools::Pools::owner(pool) == *owner) + }) + }); + if ended { + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + Ok(next) + } + fn release_render_lists(&mut self) -> Result { if self.queues.render.grow.reply.is_some() { return Err(EAGAIN); @@ -783,6 +830,22 @@ impl Registry { .find(|pair| pair.qids().contains(&qid)) .and_then(|pair| pair.oldest_spared()) } + /// Marks the interrupted group of the pair that owns `qid`. + pub(in crate::g17) fn note_render_interrupted( + &self, + qid: u8, + stamp: u32, + ) -> crate::g17::recovery::Interrupted { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .map_or(crate::g17::recovery::Interrupted::None, |pair| { + pair.note_interrupted(stamp) + }) + } pub(in crate::g17) fn visit_render_owners( &self, qid: u8, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index 2b14b4548e0db..e150a10e2c9bc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -12,6 +12,7 @@ impl Registry { packet: &Arc, ) -> Result { let mut frontier = Frontier::new(); + let compute = packet.engine() == crate::g17::job::Engine::Compute; for entry in self.render.entries.iter().flatten() { if let Some(pair) = entry.pair.as_ref() { pair.collect_dependencies( @@ -19,6 +20,7 @@ impl Registry { packet.completion.status(), packet.order.wait_through[0], &mut frontier, + compute, )?; } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { dependencies.collect( @@ -26,6 +28,7 @@ impl Registry { packet.completion.status(), packet.order.wait_through[0], &mut frontier, + compute, )?; } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs new file mode 100644 index 0000000000000..1f42d1e6f404f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/direct.rs @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Steady-state render publication without the device mutex. +//! +//! The device mutex lends the owner's installed pair out, together with its +//! free-list preparation and a parameter-buffer reference, and takes it back. +//! Between the two, command preparation and publication take only the port: +//! its render publication lock (the render work rings' single producer and +//! the publication order of attachment hazards), QoS accounting, the stamps +//! and the doorbell. While the pair is lent, the event worker skips it, so no +//! retirement can complete its commands or their free-list references. + +use super::super::{Backend, DeferredBatch}; +use crate::g17::{ + job::Packet, + object::Allocator, + port::{Port, RenderRoom}, + queue::render::{Pair, PublicationLease, RenderAddresses}, +}; +use core::sync::atomic::Ordering; +use kernel::{ + prelude::*, + sync::{Arc, MutexGuard}, +}; + +/// A pair lent out for one publication, with what that publication holds. +pub(in crate::g17::runtime) struct Loan { + pair: Option>, + preparation: Option, + buffer: u8, + lease: PublicationLease, + addresses: RenderAddresses, + uat: Arc, +} + +/// Publication host of a lent pair. Registration, first-publication and +/// growth paths never run here: the loan requires an installed, registered +/// graph that needs no growth. +struct DirectHost<'a> { + port: &'a Port, + render: &'a MutexGuard<'a, ()>, + room: RenderRoom, + generation: u64, + activation: Option<(u16, u64)>, +} + +impl crate::g17::queue::render::Host for DirectHost<'_> { + fn next_publication(&mut self) -> Result { + Ok(self.port.next_publication()) + } + fn epoch(&self) -> Result<(u64, u32)> { + Ok((self.generation, self.port.recovery_state()?)) + } + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { + Ok([self.port.stamp(qids[0])?, self.port.stamp(qids[1])?]) + } + fn qos_publish_pair( + &mut self, + owners: [crate::g17::qos::Owner; 2], + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result<[crate::g17::qos::Publication; 2]> { + self.port.qos_publish_pair(owners, scheduler, policy) + } + fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { + self.port.qos_cancel(publication) + } + fn install_pair(&mut self, _registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + Err(EAGAIN) + } + fn publish_qid(&mut self, _id: crate::g17::kick::Id) -> Result { + // An installed graph published its identities with its first command. + Ok(()) + } + fn publish_outer_pair( + &mut self, + priority: u8, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + if priority != self.room.priority() { + return Err(EINVAL); + } + self.port + .write_render_pair(self.render, &self.room, fragment, tiling, commit, late_tiling) + } + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { + // The lease leaves preparation when the pair is returned; nothing can + // observe it in between because the pair is lent. + self.activation = Some((id, generation)); + Ok(()) + } + fn publish_retained_pair( + &mut self, + priority: u8, + _fragment: &crate::g17::fw::channels::WorkSlot, + _tiling: &crate::g17::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + if priority != self.room.priority() { + return Err(EINVAL); + } + self.port.expose_render_pair(self.render, &self.room, next) + } + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + fn notify_render(&mut self, priority: u8) -> Result { + self.port + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2) + } + fn note_submission(&mut self) -> Result { + // Counted, with full effort witnessed, before the pair was lent. + Ok(()) + } + fn complete_render( + &mut self, + _owners: Option<[crate::g17::qos::Owner; 2]>, + _pool_id: u16, + _generation: u64, + _buffer_id: u8, + ) -> Result { + Err(EINVAL) + } +} + +impl crate::g17::Firmware { + /// Lends `owner`'s pair in `slot` for a publication of `packet`, or None if + /// the pair is not in the steady state the direct path requires. + fn lend_render( + &mut self, + slot: u8, + owner: u64, + packet: &Arc, + priority: u8, + generation: u64, + ) -> Result> { + if self.recovery.pending() || self.recovery.generation() != generation { + return Ok(None); + } + let Ok(entry) = self.queues.render.entry_mut(slot) else { + return Ok(None); + }; + if !entry.published + || entry.closed + || entry.constructing + || entry.preparing.is_some() + || entry.growing + || entry.parked.is_some() + || entry.pending_error.is_some() + || entry.pending_timeout + { + return Ok(None); + } + let Some(pair) = entry.pair.as_mut() else { + return Ok(None); + }; + if !pair.matches(owner, &packet.context)? + || !pair.direct_ready(&packet.context, priority) + || pair.pool().vacant() + || !matches!(pair.host_growth_target(packet), Ok(None)) + { + return Ok(None); + } + let qids = pair.qids(); + let pool = pair.pool().clone(); + let buffer_id = pair.buffer_id(); + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let (buffer, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { + Ok(preparation) => preparation, + Err(error) => { + self.queues.buffers.release(buffer)?; + return if error == EAGAIN || error == EBUSY { + Ok(None) + } else { + Err(error) + }; + } + }; + let lease = PublicationLease { + free_list_generation: preparation.generation, + buffer_token, + }; + let pair = match self.queues.render.take_pair(slot, owner, packet) { + Ok(pair) => pair, + Err(error) => { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(buffer)?; + return if error == EBUSY { Ok(None) } else { Err(error) }; + } + }; + // The selected scheduler is retained through release acknowledgement, + // as on the device-mutex path. + packet.context.mark_published(); + Ok(Some(Loan { + pair: Some(pair), + preparation: Some(preparation), + buffer, + lease, + addresses, + uat: self.uat.clone(), + })) + } + + /// Takes a lent pair back and settles what its publication held: the + /// free-list preparation is activated once published, rolled back with the + /// parameter-buffer reference otherwise. + fn return_render( + &mut self, + slot: u8, + loan: &mut Loan, + activation: Option<(u16, u64)>, + packet: &Arc, + ) -> Result { + self.queues.render.return_pair(slot, &mut loan.pair)?; + if self.queues.render.take_missed_event(slot) { + self.primary.state.shared.queue_events(); + } + let entry = self.queues.render.entry_mut(slot)?; + // A failure or timeout recorded against the lent pair is settled by the + // event worker's collector, as after any other loan. + if entry.pending_error.is_some() || entry.pending_timeout { + self.primary.state.shared.queue_events(); + } + let pair = entry.pair.as_mut().ok_or(EIO)?; + if entry.closed { + pair.release_owner(); + } + if !packet.is_published() { + pair.discard_prepared(packet)?; + } + if let Some((id, generation)) = activation { + if self.queues.render.pools.activate(id, generation).is_err() { + self.primary.state.shared.lose_device_quietly(); + } + } + let preparation = loan.preparation.take().ok_or(EIO)?; + if packet.is_published() { + self.queues.render.entry_mut(slot)?.published = true; + } else { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(loan.buffer)?; + } + Ok(()) + } +} + +impl Backend { + /// Publishes `packet` into the owner's installed pair with the device + /// mutex held only to lend the pair out and take it back. None leaves the + /// packet unpublished for the device-mutex path. + pub(in crate::g17::runtime) fn render_direct( + &self, + packet: &Arc, + slot: u8, + deferred: &mut DeferredBatch, + ) -> Option { + // A compute prefix needs the compute queue's state; leave it to the + // device-mutex path. + if packet.is_published() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || packet.order.wait_through[1].unwrap_or(0) != 0 + || self.admission_open().is_err() + { + return None; + } + let lease = self.shared.preparations.try_enter().ok()?; + let port = self.shared.port(&lease)?; + if self.shared.crashed.load(Ordering::Acquire) || port.recovery_state().ok()? != 0 { + return None; + } + // Full effort is applied by the device-mutex path when it is not already. + if !port.note_submission() { + return None; + } + match packet.completion.work_state().ok()?.acquire() { + Ok(_) => {} + Err(error) if error == EBUSY => return None, + Err(error) => return Some(Err(error)), + } + let priority = packet.context.policy().priority(); + let generation = port.generation(); + let dev = self.shared.drm().ok()?; + let mut loan = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut()?; + if !lease.is_current() { + return None; + } + match firmware.lend_render(slot, self.owner, packet, priority, generation) { + Ok(Some(loan)) => loan, + Ok(None) => return None, + Err(error) => return Some(Err(error)), + } + }; + let mut activation = None; + let result = (|| { + let pair = loan.pair.as_mut().ok_or(EIO)?; + let alloc = Allocator { + dev: &dev, + uat: &loan.uat, + }; + pair.prepare(&alloc, packet, loan.addresses)?; + let render = port.render_lock(); + if !lease.is_current() { + return Err(EBUSY); + } + let room = port.observe_render_room(&render, priority)?.ok_or(EBUSY)?; + let mut dependencies = pair.own_dependencies(self.owner, packet)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.admission.publication_barriers( + self.owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } + let mut host = DirectHost { + port, + render: &render, + room, + generation, + activation: None, + }; + let result = pair.publish( + packet.clone(), + &dependencies, + loan.lease, + &mut host, + &mut |event| deferred.push(event), + ); + activation = host.activation; + result?; + // A recovery that started meanwhile may have discarded the commands. + if port.generation() != generation || port.recovery_state()? != 0 { + return Err(EIO); + } + Ok(()) + })(); + let returned = { + let mut state = self.shared.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) => { + let returned = firmware.return_render(slot, &mut loan, activation, packet); + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { + self.shared.queue_grow(); + } + returned + } + // Shutdown joins every preparation lease before detaching the + // session, so this cannot happen while `lease` is held. + None => Err(ENODEV), + } + }; + self.shared.note_changed(); + drop(lease); + if let Err(error) = returned { + self.shared.lose_device_quietly(); + return Some(Err(error)); + } + match result { + Err(error) if (error == EAGAIN || error == EBUSY) && !packet.is_published() => None, + result => { + if result.is_ok() { + self.direct_slot.store(u32::from(slot), Ordering::Relaxed); + } + Some(result) + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index e3cb174721aaa..20569af84bc7b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -280,7 +280,7 @@ impl crate::g17::Shared { return Err(EIO); } firmware.queues.render.grow.active = None; - self.changed.notify_all(); + self.note_changed(); if !lease.is_current() || request.halt_count != firmware.recovery.generation() { continue; } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs index 4ede20d316c64..52bdd964ca4ae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -164,7 +164,7 @@ impl crate::g17::Shared { return Err(error); } let firmware = (*state).as_deref_mut().ok_or(EIO)?; - self.changed.notify_all(); + self.note_changed(); if firmware.render_grow_pending() || firmware.queues.render.take_missed_event(slot as u8) { self.queue_events(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 4713eb5b6b152..1e3354d90dbfb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -6,7 +6,7 @@ use crate::g17::{ channel::Rings, - freelist::{Lease, Owner, RenderPool}, + freelist::{Lease, Owner, RenderPool, LINGER_NS}, fw::{ channels::{ControlRecord, FreeListCompletion}, initdata::PagePoolDescriptor, @@ -49,7 +49,7 @@ impl Pools { Ok(slot) } - fn owner(pool: &RenderPool) -> Owner { + pub(super) fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), epoch: pool.epoch(), @@ -154,12 +154,12 @@ impl Pools { } /// Preflight a paired retirement before QoS or PB reference accounting. - pub(super) fn completed(&self, id: u16, generation: u64) -> Result { + pub(super) fn completed(&self, id: u16, generation: u64, linger: bool) -> Result { let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; if next.generation() != generation { return Err(EIO); } - next.complete_render()?; + next.complete_render(linger)?; Ok(next) } @@ -169,6 +169,32 @@ impl Pools { self.leases[usize::from(id)] = Some(next); } + /// Ends the linger of every idle registration that `keep` (an open owner + /// still bound to the pool) does not retain or whose linger has elapsed. + /// Returns whether a release became pending, and the next expiry delay. + pub(super) fn expire_lingers( + &mut self, + now: u64, + keep: impl Fn(&Owner) -> bool, + ) -> (bool, Option) { + let mut ended = false; + let mut next: Option = None; + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { + let Some(since) = lease.lingering() else { + continue; + }; + let age = now.saturating_sub(since); + if age >= LINGER_NS || !keep(&lease.owner()) { + lease.end_linger(); + ended = true; + } else { + let delay = LINGER_NS - age; + next = Some(next.map_or(delay, |next| next.min(delay))); + } + } + (ended, next) + } + pub(super) fn pending(&self) -> bool { self.leases .iter() @@ -186,7 +212,7 @@ impl Pools { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { lease.observe_releases(consumer, producer)?; } Ok(()) @@ -199,7 +225,7 @@ impl Pools { init: &InitData, mut announce: impl FnMut() -> Result, ) -> Result { - for lease in self.leases.iter_mut().flatten() { + for lease in self.leases[FIRST_RENDER_POOL..].iter_mut().flatten() { let Some(release) = lease.pending_release() else { continue; }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index e2034c3176595..62ca7895a86f8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -426,7 +426,7 @@ impl super::super::Backend { } firmware.render_grow_ready() }; - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -548,7 +548,7 @@ impl super::super::Backend { .release(id)?; } } - self.shared.changed.notify_all(); + self.shared.note_changed(); if grow_ready { self.shared.queue_grow(); } @@ -593,6 +593,10 @@ impl super::super::Backend { pub(super) fn ensure_render(&self, packet: &Arc, slot: u8, sleep: bool) -> Result { self.build_render_pair(slot, packet, false, sleep)?; + // An owner has exactly one pair (kick.rs asserts it): no sibling to warm. + if crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER == 1 { + return Ok(()); + } let sibling = { let state = self.shared.state.lock(); (*state) @@ -711,7 +715,7 @@ impl super::super::Backend { .ok_or(EIO)? .release_owner(); } - self.shared.changed.notify_all(); + self.shared.note_changed(); let published = (|| { let entry = firmware.queues.render.entry(slot)?; if entry.pending_error.is_some() || entry.pending_timeout { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index b77fc13b7ed2f..1ef6742b689a7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -8,22 +8,18 @@ use crate::g17::{ queue::admission::{RenderLease, Request}, }; use core::sync::atomic::Ordering; -use kernel::{ - prelude::*, - sync::Arc, - time::{delay::fsleep, Delta}, -}; +use kernel::{prelude::*, sync::Arc}; /// Internal result of a publication attempted from the submitting ioctl that /// met contention before anything became visible: the scheduler publishes the /// command instead. Never returned to userspace. -pub(super) const FALLBACK: Error = EINPROGRESS; +pub(in crate::g17::runtime) const FALLBACK: Error = EINPROGRESS; impl Backend { /// The submitting ioctl holds the logical queue's mutex and never sleeps /// for contention or a closed preparation gate: it returns FALLBACK, and /// the scheduler's worker publishes the command and owns the waits. - pub(super) fn enter_preparation(&self, sleep: bool) -> Result { + pub(in crate::g17::runtime) fn enter_preparation(&self, sleep: bool) -> Result { if sleep { return self.shared.preparations.enter(false); } @@ -48,18 +44,20 @@ impl Backend { /// Reserve identities before attaching a lease to the packet. An empty /// logical slot must not hide reusable physical channels when the shared /// queue-ID namespace is full. + /// Also reports whether the owner's installed pair is ready to take the + /// command, so that the publication skips graph construction. fn acquire_render_with_ids( &self, packet: &Arc, mut next: Option, sleep: bool, - ) -> Result { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; + ) -> Result<(RenderLease, bool)> { + let pool = &self.admission; let started = super::super::now_ns(); let deadline_ns = started.saturating_add(u64::from(crate::g17::job::TIMEOUT_MS) * 1_000_000); + // A capacity refusal samples the render domain and retries once; only + // a refusal after that sample waits, so no release can be missed. + let mut observed = None; loop { self.admission_open()?; if super::super::now_ns() >= deadline_ns { return Err(EBUSY); } @@ -97,7 +95,7 @@ impl Backend { .reclaim_unused_render_spare(Some(self.owner), false)? }; if reclaimed.is_none() { - self.wait_render_capacity(started, sleep)?; + self.wait_render_capacity(&mut observed, started, sleep)?; } drop(reclaimed); continue; @@ -109,7 +107,10 @@ impl Backend { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; match firmware.queues.ensure_render_ids(slot, self.owner) { - Ok(()) => return Ok(lease), + Ok(()) => { + let ready = firmware.queues.render_ready(slot, self.owner, &self.context)?; + return Ok((lease, ready)); + } Err(error) if error == ENOSPC => { let reclaimed = firmware.queues.reclaim_unused_render_spare(None, true)?; if reclaimed.is_some() { @@ -131,7 +132,7 @@ impl Backend { let ready = firmware.queues.has_render_ids(self.owner); drop(state); if reclaimed.is_none() && !ready { - self.wait_render_capacity(started, sleep)?; + self.wait_render_capacity(&mut observed, started, sleep)?; } drop(reclaimed); } @@ -140,31 +141,178 @@ impl Backend { } } - /// A pair is borrowed, still settling, or waiting for the release of the - /// scheduler it last ran for to be consumed and the pair parked. The event - /// worker observes both, so kick it on every poll; time out only after the - /// same bound as a capacity wait. - fn wait_render_busy(&self, since: &mut Option, sleep: bool) -> Result { + /// Scheduler admission of a render packet. A work state of its context or + /// a pair admission it lacks becomes a fence that the return providing it + /// signals, so run_job finds both ready instead of waiting for them. + /// Errors, and anything else publication may wait for, are left to it. + pub(in crate::g17::runtime) fn admit_render(&self, packet: &Arc) -> Option { + if packet.early_result().is_some() + || packet.cancel_requested() + || packet.completion.status().get() != 0 + || self.admission_open().is_err() + { + return None; + } + // Each check runs once more after sampling its domain, so the common + // case takes no domain lock and no return between check and wait is lost. + let work = packet.completion.work_state().ok()?; + loop { + match work.acquire() { + Err(error) if error == EBUSY => { + let states = work.returns().observe(); + match work.acquire() { + Err(error) if error == EBUSY => match work.returns().wait(states) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + }, + Err(_) => return None, + Ok(_) => {} + } + } + Err(_) => return None, + Ok(_) => {} + } + let request = Request::of(packet); + if self.admission.admits_render(self.owner, request) { + return None; + } + let observed = self.shared.renders.observe(); + if self.admission.admits_render(self.owner, request) { + return None; + } + self.shared.queue_events(); + match self.shared.renders.wait(observed) { + Ok(Some(fence)) => return Some(fence), + Ok(None) => continue, + Err(_) => return None, + } + } + } + + /// Conditions which end a publication that has not become visible yet. + fn publication_open(&self, packet: &Packet) -> Result { + self.admission_open()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + Ok(()) + } + + /// Blocks on `domain` until a notification after `observed`, or the + /// deadline, which only bounds the wait. Slices of at most 10 ms let the + /// publisher observe cancellation, which has no notification here. + fn wait_domain( + &self, + domain: &crate::g17::availability::Availability, + observed: u64, + deadline_ns: u64, + packet: &Packet, + ) -> Result { + self.publication_open(packet)?; + let remaining = deadline_ns.saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(ETIMEDOUT); + } + domain.block(observed, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// A pair is borrowed, still settling, waiting for firmware room or for the + /// release of the scheduler it last ran for. Every pair return, retirement + /// and teardown step signals the render domain, and each event pass does + /// while a publisher waits; the deadline is only the failure bound. + fn wait_render_busy( + &self, + since: &mut Option, + observed: &mut Option, + sleep: bool, + packet: &Packet, + ) -> Result { if !sleep { return Err(FALLBACK); } let now = super::super::now_ns(); - if now.saturating_sub(*since.get_or_insert(now)) >= super::super::WORK_RING_WAIT_NS { - return Err(ETIMEDOUT); - } + let deadline = since.get_or_insert(now).saturating_add(super::super::WORK_RING_WAIT_NS); + // Sample, then retry once: only a refusal after the sample waits. + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; self.shared.queue_events(); - fsleep(Delta::from_micros(200)); - Ok(()) + self.wait_domain(&self.shared.renders, sample, deadline, packet) } - /// A closing owner may still hold its pair until exact retirement. Wait - /// outside both the device mutex and preparation gate so the event worker - /// and recovery can return capacity. No packet is firmware-visible here. - fn wait_render_capacity(&self, started: u64, sleep: bool) -> Result { + /// Render capacity is short: return what idle owners hold, then block until + /// a pair, lease or owner returns. No packet is firmware-visible here, and + /// neither the device mutex nor a preparation lease is held while waiting. + fn wait_render_capacity(&self, observed: &mut Option, started: u64, sleep: bool) -> Result { if !sleep { return Err(FALLBACK); } - self.wait_capacity(started) + if self.reclaim_render_capacity()? { + *observed = None; + return Ok(()); + } + let Some(sample) = observed.take() else { + *observed = Some(self.shared.renders.observe()); + return Ok(()); + }; + self.shared.queue_events(); + self.admission_open()?; + let remaining = started + .saturating_add(super::super::WORK_RING_WAIT_NS) + .saturating_sub(super::super::now_ns()); + if remaining == 0 { + return Err(EBUSY); + } + self.shared + .renders + .block(sample, remaining.div_ceil(1_000_000).min(10) as u32, false) + } + + /// Gives idle render backing back under capacity pressure. Returns whether + /// anything was returned, in which case the caller retries at once. + fn reclaim_render_capacity(&self) -> Result { + let mut deferred = DeferredBatch::publication()?; + let result: Result = (|| { + let lease = self.shared.preparations.try_enter()?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() || self.shared.crashed.load(Ordering::Acquire) { + return Err(EAGAIN); + } + if firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Ok(false); + } + firmware.queues.observe_pool_releases(&firmware.init, &mut deferred)?; + firmware.queues.reclaim_idle_render_pools(&firmware.init, &mut deferred)?; + let mut published = 0; + if !firmware.render_control_backpressured() { + (published, _) = firmware.queues.vacate_idle_pools( + &firmware.init, + 0, + super::super::now_ns(), + 0, + )?; + if published != 0 { + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + } + Ok(published != 0 + || !deferred.render.is_empty() + || !deferred.vacated.is_empty() + || !deferred.bindings.is_empty()) + })(); + deferred.finish(); + match result { + Err(error) if error == EAGAIN => Ok(false), + result => result, + } } fn render_with_lease( @@ -185,14 +333,44 @@ impl Backend { // A publication from the submitting ioctl (an early lease) never // sleeps for contention; it falls back to the scheduler instead. let sleep = early.is_none(); - let admission = self.acquire_render_with_ids(packet, early, sleep)?; + let mut early = early; + // The pair this owner last published to directly: a lease on it goes + // straight to the direct path, whose lend checks everything the + // identity check would. + let known = self.direct_slot.load(Ordering::Relaxed); + let lease = match early.take() { + Some(lease) => Some(lease), + None if known != 0 => self + .admission + .try_acquire_render(self.owner, Request::of(packet)) + .ok() + .flatten(), + None => None, + }; + let early = match lease { + Some(lease) if u32::from(lease.slot()) == known => { + let slot = lease.slot(); + packet.completion.attach_render_lease(lease)?; + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + packet.completion.detach_render_lease() + } + lease => lease, + }; + let (admission, mut ready) = self.acquire_render_with_ids(packet, early, sleep)?; let slot = admission.slot(); packet.completion.attach_render_lease(admission)?; - let mut state_polls = 0; - let mut state_delay = 20i64; + if ready { + if let Some(result) = self.render_direct(packet, slot, deferred) { + return result; + } + } let mut busy_since = None; - let mut recovery_polls = 0; let capacity_started = super::super::now_ns(); + let deadline_ns = capacity_started.saturating_add(super::super::WORK_RING_WAIT_NS); + let mut seen = None; + let mut states = None; loop { if packet.cancel_requested() { return Err(ECANCELED); @@ -223,54 +401,66 @@ impl Backend { if !sleep { return Err(FALLBACK); } - if state_polls == 1000 { - return Err(ETIMEDOUT); + // Every node of this context belongs to unretired work: + // sample, retry once, then wait for the next returned one. + let work = packet.completion.work_state()?; + match states.take() { + None => states = Some(work.returns().observe()), + Some(sample) => { + self.wait_domain(work.returns(), sample, deadline_ns, packet)? + } } - state_polls += 1; - fsleep(Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); continue; } result => { result?; } } - match self.ensure_render(packet, slot, sleep) { + // A pair found ready under the identity check needs no build; any + // retry revalidates through ensure_render. + let ensured = if core::mem::take(&mut ready) { + Ok(()) + } else { + self.ensure_render(packet, slot, sleep) + }; + match ensured { Err(error) if error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, capacity_started, sleep)?; continue; } Err(error) if error == EAGAIN => continue, Err(error) if error == EBUSY => { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; continue; } result => result?, } match self.prepare_render(slot, packet, deferred, sleep) { Err(error) if !packet.is_published() && error == ENOSPC => { - self.wait_render_capacity(capacity_started, sleep)?; + self.wait_render_capacity(&mut seen, capacity_started, sleep)?; continue; } Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { - let recovering = { + let (recovering, gate) = { let state = self.shared.state.lock(); let firmware = (*state).as_deref().ok_or(ENODEV)?; - firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + ( + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0, + self.shared.preparations.snapshot(), + ) }; if recovering { if !sleep { return Err(FALLBACK); } - if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { - return Err(ETIMEDOUT); - } - recovery_polls += 1; + // Recovery closes the preparation gate before touching + // any queue; the next pass blocks in enter() until it + // reopens. The worker raises the closure. self.shared.queue_events(); - fsleep(Delta::from_millis(1)); + self.shared.preparations.wait_closure(gate, deadline_ns)?; } else if error == EBUSY { - self.wait_render_busy(&mut busy_since, sleep)?; + self.wait_render_busy(&mut busy_since, &mut seen, sleep, packet)?; } else if !sleep { return Err(FALLBACK); } @@ -284,19 +474,9 @@ impl Backend { if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { return false; } - let admission = { - let state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref() else { - return false; - }; - match firmware - .queues - .admission - .try_acquire_render(self.owner, Request::of(packet)) - { - Ok(Some(lease)) => lease, - _ => return false, - } + let admission = match self.admission.try_acquire_render(self.owner, Request::of(packet)) { + Ok(Some(lease)) => lease, + _ => return false, }; if !packet.begin_publication() { packet.completion.fail_unpublished(ECANCELED); @@ -448,15 +628,28 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } + let mut dependencies = self.queues.render_dependencies(owner, packet, state)?; + // Every render publication, with or without the device mutex, holds the + // port's render lock: the render work rings have one host producer at a + // time and attachment overlaps are ordered by publication order. + let port = self.port.clone(); + let _render = port.render_lock(); // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. All host producers share this lock. + // either inner queue becomes visible. let priority = packet.context.policy().priority(); if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } - let dependencies = self.queues.render_dependencies(owner, packet, state)?; + if let Some(footprint) = packet.footprint.as_ref() { + dependencies.hazard_count = self.queues.admission.publication_barriers( + owner, + slot, + footprint, + &mut dependencies.hazards, + )?; + } let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; // An owner change taken from a live owner which closed meanwhile: its // first command waits until that scheduler's release has been consumed. diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs index eb04f80361955..e54b1f4638f58 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -46,6 +46,14 @@ impl RenderState { Ok(true) } + /// Some render pair runs another context's commands. + pub(in crate::g17::runtime) fn foreign_in_flight(&self, context: &Arc) -> bool { + self.entries.iter().flatten().any(|entry| { + !entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) + && entry.pair.as_ref().is_some_and(|pair| pair.in_flight()) + }) + } + pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { context.render_pools_released(|id, control| self.pools.released(id, control)) && self diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index f8020e8962f84..16bd5828265a4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -115,7 +115,7 @@ impl Host for Service<'_> { self.compute .iter() .flatten() - .filter_map(|entry| entry.queue.as_ref()) + .filter_map(|entry| entry.queue.as_deref()) .all(|queue| queue.idle_for_context(context) && !queue.switching_from(context)) } fn flists_released(&self, context: &Arc) -> bool { @@ -190,10 +190,17 @@ impl Firmware { teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); + let mut consumed = false; while teardown .take_consumed(consumer, producer, &mut service)? .is_some() - {} + { + consumed = true; + } + if consumed { + // A channel whose last scheduler release is consumed can change owner. + self.primary.state.shared.channels.notify(); + } Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/sync_input.c b/drivers/gpu/drm/asahi/g17/sync_input.c new file mode 100644 index 0000000000000..19d5b8b92f934 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include "sync_input.h" +#include "sync_scope.h" + +static int collect_leaf(struct dma_fence *leaf, + struct asahi_g17_fence_leaf *out, size_t capacity, + size_t *count) +{ + if (*count == capacity) + return -ENOSPC; + out[*count].fence = dma_fence_get(leaf); + out[*count].status = dma_fence_get_status(leaf); + ++*count; + return 0; +} + +/* Called without the device mutex: status polling may run fence callbacks. */ +int asahi_g17_collect_fences(struct dma_fence *head, + struct asahi_g17_fence_leaf *out, size_t capacity) +{ + struct dma_fence *pending[128]; + size_t count = 0, depth = 1; + int error = 0; + + pending[0] = dma_fence_get(head); + while (depth && !error) { + struct dma_fence *node = pending[--depth]; + struct dma_fence_unwrap cursor; + struct dma_fence *leaf; + + if (asahi_g17_is_scope(node)) { + struct dma_fence *children[2]; + unsigned int n = asahi_g17_scope_children(node, children), i; + if (!n) { + /* Preserve a failed scope as a failed leaf, never drop its error. */ + if (dma_fence_get_status(node) < 0) + error = collect_leaf(node, out, capacity, &count); + } else if (depth + n > ARRAY_SIZE(pending)) { + for (i = 0; i < n; ++i) + dma_fence_put(children[i]); + error = -ENOSPC; + } else { + for (i = 0; i < n; ++i) + pending[depth++] = children[i]; + } + dma_fence_put(node); + continue; + } + + dma_fence_unwrap_for_each(leaf, &cursor, node) { + if (asahi_g17_is_scope(leaf)) { + if (depth == ARRAY_SIZE(pending)) + error = -ENOSPC; + else + pending[depth++] = dma_fence_get(leaf); + } else if (dma_fence_is_container(leaf)) { + error = -EOPNOTSUPP; + } else { + error = collect_leaf(leaf, out, capacity, &count); + } + if (error) + break; + } + /* A bounded early exit still owns the iterator's current chain. */ + dma_fence_put(cursor.chain); + dma_fence_put(node); + } + while (depth) + dma_fence_put(pending[--depth]); + if (error) { + while (count) + dma_fence_put(out[--count].fence); + return error; + } + return count; +} + +void asahi_g17_put_fence(struct dma_fence *fence) +{ + dma_fence_put(fence); +} diff --git a/drivers/gpu/drm/asahi/g17/sync_input.h b/drivers/gpu/drm/asahi/g17/sync_input.h new file mode 100644 index 0000000000000..2b6e80f4b7f66 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_INPUT_H +#define ASAHI_G17_SYNC_INPUT_H + +#include + +struct dma_fence; + +struct asahi_g17_fence_leaf { + struct dma_fence *fence; + int status; +}; + +int asahi_g17_collect_fences(struct dma_fence *head, + struct asahi_g17_fence_leaf *out, size_t capacity); +void asahi_g17_put_fence(struct dma_fence *fence); + +#endif diff --git a/drivers/gpu/drm/asahi/g17/sync_input.rs b/drivers/gpu/drm/asahi/g17/sync_input.rs new file mode 100644 index 0000000000000..1cd9a9ba761aa --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_input.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded input-fence components. Collection happens before the device mutex, +//! because polling a container's status can signal it and run callbacks. + +use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; + +#[repr(C)] +pub(super) struct Leaf { + pub(super) fence: *mut bindings::dma_fence, + pub(super) status: i32, +} + +unsafe extern "C" { + fn asahi_g17_collect_fences(head: *mut bindings::dma_fence, + out: *mut Leaf, capacity: usize) -> i32; + fn asahi_g17_put_fence(fence: *mut bindings::dma_fence); +} + +impl Drop for Leaf { + fn drop(&mut self) { + // SAFETY: Successful collection transfers exactly one owned reference. + unsafe { asahi_g17_put_fence(self.fence) }; + } +} + +/// Allocation, unsupported containers, or a large prefix retain normal waits. +pub(super) fn collect(input: &Fence) -> Option> { + let mut capacity = 4; + loop { + let mut leaves = KVec::with_capacity(capacity, GFP_KERNEL).ok()?; + // SAFETY: The input owns its fence. The C helper writes at most capacity + // fully initialized records and releases every reference on failure. + let count = unsafe { + asahi_g17_collect_fences(input.raw(), leaves.as_mut_ptr(), capacity) + }; + if count == ENOSPC.to_errno() && capacity < 128 { + capacity = (capacity * 4).min(128); + continue; + } + if count < 0 { return None; } + // SAFETY: The helper returned this many initialized, owned records. + unsafe { leaves.set_len(count as usize) }; + return Some(leaves); + } +} + + +use super::fence::Submission; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::{new_mutex, sync::{Arc, Mutex}}; + +pub(super) struct Source { + pub(super) submission: Arc, +} + +/// The counter is independent of a physical queue's allocation and survives +/// processor stop until the last consumer releases its binding loan. +pub(super) struct Hold(Arc); +impl Hold { + fn new(readers: &Arc) -> Result { + readers.fetch_update(Ordering::AcqRel, Ordering::Acquire, + |count| count.checked_add(1)).map_err(|_| EOVERFLOW)?; + Ok(Self(readers.clone())) + } +} +impl Drop for Hold { + fn drop(&mut self) { self.0.fetch_sub(1, Ordering::AcqRel); } +} + +/// A new owner starts with no loans. Acquiring a loan requires exclusive access +/// and arms the hint; eligibility queries stay read-only. A validated owner +/// transition can clear the hint after all independent loans have been released. +#[repr(transparent)] +pub(super) struct ReaderHint { + possible: bool, +} +impl ReaderHint { + pub(super) const fn new() -> Self { + Self { possible: false } + } + pub(super) fn hold(&mut self, count: &Arc) -> Result { + let hold = Hold::new(count)?; + self.possible = true; + Ok(hold) + } + pub(super) fn has_readers(&self, count: &AtomicU32) -> bool { + self.possible && count.load(Ordering::Acquire) != 0 + } + pub(super) fn reset_for_owner(&mut self, count: &AtomicU32) -> Result { + if self.has_readers(count) { return Err(EBUSY); } + self.possible = false; + Ok(()) + } +} + +#[pin_data] +pub(super) struct Sources { + inputs: KVec, + publication_waits: KVec, + pub(super) sources: KVec, + #[pin] + holds: Mutex>, +} +impl Sources { + pub(super) fn new(inputs: KVec, sources: KVec, + publication_waits: KVec) -> Result> { + CONVERTED.fetch_add(1, Ordering::Relaxed); + if !publication_waits.is_empty() { PENDING.fetch_add(1, Ordering::Relaxed); } + Arc::pin_init(pin_init!(Self { + inputs, sources, publication_waits, + holds <- new_mutex!(KVec::new(), "G17 source binding loans"), + }), GFP_KERNEL) + } + pub(super) fn contains_input(&self, input: &Fence) -> bool { + self.inputs.iter().any(|fence| fence.raw() == input.raw()) + } + pub(super) fn publication_waits(&self) -> &[Fence] { &self.publication_waits } + pub(super) fn check(&self) -> Result { + for source in &self.sources { source.submission.source_status()?; } + Ok(()) + } + pub(super) fn set_holds(&self, holds: KVec) { + let old = core::mem::replace(&mut *self.holds.lock(), holds); + drop(old); + } + pub(super) fn release(&self) { self.set_holds(KVec::new()); } +} + + +static CONVERTED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static PUBLISHED: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +static PENDING: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(super) fn pending_count() -> u64 { PENDING.load(Ordering::Relaxed) } +pub(super) fn note_published() { PUBLISHED.fetch_add(1, Ordering::Relaxed); } +pub(super) fn counters() -> (u64, u64) { + (CONVERTED.load(Ordering::Relaxed), PUBLISHED.load(Ordering::Relaxed)) +} + +/// A revalidated compute completion dependency for a direct work descriptor. +#[derive(Clone, Copy)] +pub(super) struct NativeWait { + pub(super) sequence: u64, + pub(super) dependency: super::fw::kick::KickDependency, + pub(super) stamp_va: u64, + pub(super) event_slot: u32, +} +impl NativeWait { + pub(super) fn bytes(&self, consumer_stamp: u32, uuid: u64) -> [u8; 0x48] { + let mut out = [0; 0x48]; + let timestamp = self.dependency.word() & ((1u64 << 40) - 1); + for (offset, value) in [ + (0, 4u32), + (0x14, u32::from(self.dependency.qid())), + (0x18, super::fw::stamp_value(timestamp as u32)), + (0x24, self.event_slot), + (0x28, consumer_stamp), + (0x34, 1), + ] { + out[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + for (offset, value) in [(4, self.stamp_va), (0xc, timestamp), (0x2c, uuid)] { + out[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); + } + out + } +} + +static NATIVE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +pub(super) fn note_native(count: usize) { NATIVE.fetch_add(count as u64, Ordering::Relaxed); } +pub(super) fn native_count() -> u64 { NATIVE.load(Ordering::Relaxed) } + +#[kunit_tests(asahi_g17_input_wait)] +mod tests { + use super::*; + use crate::g17::fw::kick::{KickDependency, KickTimestamp}; + + #[test] + fn completion_record_preserves_distinct_event_and_queue_domains() { + let wait = NativeWait { + sequence: 1, + dependency: KickDependency::new(37, + KickTimestamp::new(0x12_3456_789a).unwrap()).unwrap(), + stamp_va: 0x1122_3344_5566_7788, + event_slot: 7, + }; + // Protocol fixture includes the unaligned stamp pointer, full 40-bit + // timestamp, wrapped 24-bit stamp and independent legacy event slot. + let expected: [u8; 0x48] = [ + 4, 0, 0, 0, 0x88, 0x77, 0x66, 0x55, + 0x44, 0x33, 0x22, 0x11, 0x9a, 0x78, 0x56, 0x34, + 0x12, 0, 0, 0, 37, 0, 0, 0, + 0, 0x9a, 0x78, 0x56, 0, 0, 0, 0, + 0, 0, 0, 0, 7, 0, 0, 0, + 0, 0xcc, 0xbb, 0xaa, 8, 7, 6, 5, + 4, 3, 2, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + ]; + assert_eq!(wait.bytes(0xaabb_cc00, 0x0102_0304_0506_0708), expected); + } + + #[test] + fn failed_loan_acquisition_never_wraps_or_releases_another_reader() { + let readers = Arc::new(AtomicU32::new(u32::MAX - 1), GFP_KERNEL).unwrap(); + let last = Hold::new(&readers).unwrap(); + assert!(matches!(Hold::new(&readers), Err(error) if error == EOVERFLOW)); + assert_eq!(readers.load(Ordering::Acquire), u32::MAX); + drop(last); + assert_eq!(readers.load(Ordering::Acquire), u32::MAX - 1); + } + + #[test] + fn loan_outlives_original_queue_counter_reference() { + let readers = Arc::new(AtomicU32::new(0), GFP_KERNEL).unwrap(); + let mut hint = ReaderHint::new(); + let first = hint.hold(&readers).unwrap(); + let second = hint.hold(&readers).unwrap(); + drop(readers); + assert_eq!(second.0.load(Ordering::Acquire), 2); + drop(first); + assert_eq!(second.0.load(Ordering::Acquire), 1); + drop(second); + } + + #[test] + fn reader_guard_tracks_partial_release_and_rearms_after_zero() { + let readers = Arc::new(AtomicU32::new(0), GFP_KERNEL).unwrap(); + let mut hint = ReaderHint::new(); + assert!(!hint.has_readers(&readers)); + let first = hint.hold(&readers).unwrap(); + let second = hint.hold(&readers).unwrap(); + assert!(hint.has_readers(&readers)); + assert!(matches!(hint.reset_for_owner(&readers), Err(error) if error == EBUSY)); + drop(first); + assert!(hint.has_readers(&readers)); + drop(second); + assert!(!hint.has_readers(&readers)); + hint.reset_for_owner(&readers).unwrap(); + let next = hint.hold(&readers).unwrap(); + assert!(hint.has_readers(&readers)); + drop(next); + assert!(!hint.has_readers(&readers)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/sync_output.c b/drivers/gpu/drm/asahi/g17/sync_output.c new file mode 100644 index 0000000000000..08b4a95f26c9b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include "sync_output.h" +#include "sync_scope.h" + +struct asahi_g17_signal_target { + struct drm_syncobj *syncobj; + struct dma_fence_chain *chain; + u64 point; +}; + +struct asahi_g17_signal_bundle { + size_t count; + size_t anchor; + bool published; + struct asahi_g17_signal_target targets[]; +}; + +void asahi_g17_signals_free(struct asahi_g17_signal_bundle *bundle) +{ + size_t i; + + if (!bundle) + return; + for (i = 0; i < bundle->count; ++i) { + struct asahi_g17_signal_target *target = &bundle->targets[i]; + if (target->chain) + dma_fence_chain_free(target->chain); + if (target->syncobj) + drm_syncobj_put(target->syncobj); + } + kfree(bundle); +} + +/* The Rust caller supplies its validated, immutable snapshot, never a user pointer. */ +struct asahi_g17_signal_bundle * +asahi_g17_signals_new(const struct asahi_g17_signal_spec *syncs, + size_t count, size_t anchor, int *error) +{ + struct asahi_g17_signal_bundle *bundle; + size_t i; + + *error = -EINVAL; + if (!count || anchor >= count || !syncs[anchor].point) + return NULL; + *error = -ENOMEM; + bundle = kzalloc(struct_size(bundle, targets, count), GFP_KERNEL); + if (!bundle) + return NULL; + bundle->count = count; + bundle->anchor = anchor; + for (i = 0; i < count; ++i) { + struct asahi_g17_signal_target *target = &bundle->targets[i]; + target->syncobj = syncs[i].syncobj; + if (target->syncobj) + drm_syncobj_get(target->syncobj); + if (!target->syncobj) { + *error = -ENOENT; + goto fail; + } + target->point = syncs[i].point; + if (syncs[i].timeline) { + target->chain = dma_fence_chain_alloc(); + if (!target->chain) + goto fail; + } + if (i != anchor && target->syncobj == bundle->targets[anchor].syncobj) { + *error = -EINVAL; + goto fail; + } + } + /* The anchor may follow the other outputs. Reject aliases in either order. */ + for (i = 0; i < count; ++i) { + if (i != anchor && bundle->targets[i].syncobj == bundle->targets[anchor].syncobj) { + *error = -EINVAL; + goto fail; + } + } + *error = 0; + return bundle; +fail: + asahi_g17_signals_free(bundle); + return NULL; +} + +static void asahi_g17_signal_install(struct asahi_g17_signal_target *target, + struct dma_fence *fence) +{ + if (target->chain) { + drm_syncobj_add_point(target->syncobj, target->chain, fence, target->point); + target->chain = NULL; + } else { + drm_syncobj_replace_fence(target->syncobj, fence); + } +} + +int asahi_g17_signals_publish(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *submitted) +{ + struct dma_fence *previous, *scope; + size_t i; + + if (bundle->published) + return -EINVAL; + previous = drm_syncobj_fence_get(bundle->targets[bundle->anchor].syncobj); + scope = asahi_g17_scope_create(previous, submitted, + bundle->targets[bundle->anchor].point); + dma_fence_put(previous); + if (!scope) + return -ENOMEM; + /* Only exported completion includes earlier work. GPU execution gets no new wait. */ + asahi_g17_signal_install(&bundle->targets[bundle->anchor], scope); + previous = drm_syncobj_fence_get(bundle->targets[bundle->anchor].syncobj); + asahi_g17_scope_bind(scope, previous); + dma_fence_put(previous); + for (i = 0; i < bundle->count; ++i) + if (i != bundle->anchor) + asahi_g17_signal_install(&bundle->targets[i], scope); + bundle->published = true; + dma_fence_put(scope); + return 0; +} + +/* Allocation or validation failed before enqueue: install an error promise without allocating. */ +void asahi_g17_signals_fail(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *failure) +{ + size_t i; + + if (bundle->published) + return; + for (i = 0; i < bundle->count; ++i) + asahi_g17_signal_install(&bundle->targets[i], failure); + bundle->published = true; +} diff --git a/drivers/gpu/drm/asahi/g17/sync_output.h b/drivers/gpu/drm/asahi/g17/sync_output.h new file mode 100644 index 0000000000000..a752d9ceaf4b0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_OUTPUT_H +#define ASAHI_G17_SYNC_OUTPUT_H +#include + +struct dma_fence; +struct drm_syncobj; +struct asahi_g17_signal_bundle; +struct asahi_g17_signal_spec { + struct drm_syncobj *syncobj; + u64 point; + u32 timeline; +}; + +struct asahi_g17_signal_bundle * +asahi_g17_signals_new(const struct asahi_g17_signal_spec *specs, + size_t count, size_t anchor, int *error); +void asahi_g17_signals_free(struct asahi_g17_signal_bundle *bundle); +int asahi_g17_signals_publish(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *submitted); +void asahi_g17_signals_fail(struct asahi_g17_signal_bundle *bundle, + struct dma_fence *failure); +#endif diff --git a/drivers/gpu/drm/asahi/g17/sync_output.rs b/drivers/gpu/drm/asahi/g17/sync_output.rs new file mode 100644 index 0000000000000..928ebe2319394 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_output.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! Atomic preparation of a queue completion scope and its exported signals. + +use kernel::{bindings, dma_fence::{Fence, RawDmaFence}, prelude::*}; + +// Wire values also documented in include/uapi/drm/asahi_drm.h. Keeping these +// local permits building this module against an older kernel's Rust bindings. +pub(crate) const SYNC_CUMULATIVE: u32 = 2; +pub(crate) const FEATURE_CUMULATIVE_SIGNALS: u64 = 1 << 29; + +unsafe extern "C" { + fn asahi_g17_signals_new(syncs: *const SignalSpec, count: usize, anchor: usize, + error: *mut i32) -> *mut core::ffi::c_void; + fn asahi_g17_signals_free(bundle: *mut core::ffi::c_void); + fn asahi_g17_signals_publish(bundle: *mut core::ffi::c_void, + current: *mut bindings::dma_fence) -> i32; + fn asahi_g17_signals_fail(bundle: *mut core::ffi::c_void, + failure: *mut bindings::dma_fence); +} + +#[repr(C)] +struct SignalSpec { + syncobj: *mut core::ffi::c_void, + point: u64, + timeline: u32, +} + +/// Owns syncobj references and preallocated chain nodes; never owns user pointers. +pub(crate) struct SignalBundle(*mut core::ffi::c_void); + +impl SignalBundle { + /// C retains references before the borrowed sync items can be dropped. + pub(crate) fn new(syncs: &[crate::file::SyncItem], anchor: usize) -> Result + { + let mut specs = KVec::with_capacity(syncs.len(), GFP_KERNEL)?; + for sync in syncs { + specs.push(SignalSpec { + syncobj: sync.syncobj.as_raw().cast(), point: sync.timeline_value, + timeline: u32::from(sync.timeline), + }, GFP_KERNEL)?; + } + let mut error = 0; + // SAFETY: All entries borrow live sync objects; C copies the entries and + // acquires its own references before returning. + let raw = unsafe { asahi_g17_signals_new( + specs.as_ptr(), specs.len(), anchor, &mut error) }; + if raw.is_null() { return Err(Error::from_errno(error)); } + Ok(Self(raw)) + } + + pub(crate) fn publish(&mut self, current: &Fence) -> Result { + // SAFETY: The bundle and current fence are owned for this call; C takes + // its own references on publication and reports failure before any store. + let error = unsafe { asahi_g17_signals_publish(self.0, current.raw()) }; + if error == 0 { Ok(()) } else { Err(Error::from_errno(error)) } + } + + pub(crate) fn fail(&mut self, failure: &Fence) { + // SAFETY: Live owned arguments; this only consumes uninstalled chain nodes. + unsafe { asahi_g17_signals_fail(self.0, failure.raw()) }; + } +} + +impl Drop for SignalBundle { + fn drop(&mut self) { + // SAFETY: Exactly one owner; published syncobjs retain their own fences. + unsafe { asahi_g17_signals_free(self.0) }; + } +} diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.c b/drivers/gpu/drm/asahi/g17/sync_scope.c new file mode 100644 index 0000000000000..a3153f3122b94 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_scope.c @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sync_scope.h" +#include "sync_input.h" + +int asahi_g17_completion_fence_status(struct dma_fence *fence) +{ + /* These private fences have no signaled callback. A pending flag is + * sufficient to keep the frontier member; a concurrent signal can only + * defer pruning until the next submission. Read terminal errors through + * the normal locked API after observing the signalled flag. + */ + if (!dma_fence_test_signaled_flag(fence)) + return 0; + return dma_fence_get_status(fence); +} + +struct asahi_g17_scope; +struct asahi_g17_scope_callback { + struct dma_fence_cb callback; + struct asahi_g17_scope *scope; +}; + +struct asahi_g17_scope { + struct dma_fence base; + atomic_t pending; + struct dma_fence *children[2]; + struct asahi_g17_scope_callback callbacks[2]; + struct irq_work work; + struct llist_node ready; + /* Identifies the exact anchor point this scope replaced. */ + u64 anchor_context; + u64 anchor_point; +}; + +static const struct dma_fence_ops scope_ops; +static DEFINE_PER_CPU(struct llist_head *, active_scopes); +#define SCOPE_ATOMIC_BATCH 64 + +bool asahi_g17_is_scope(struct dma_fence *fence) +{ + return fence && fence->ops == &scope_ops; +} + +static struct asahi_g17_scope *to_scope(struct dma_fence *fence) +{ + return container_of(fence, struct asahi_g17_scope, base); +} + +static const char *scope_driver_name(struct dma_fence *fence) +{ + return "asahi"; +} + +static const char *scope_timeline_name(struct dma_fence *fence) +{ + return "g17-queue-scope"; +} + +static bool scope_signaled(struct dma_fence *fence) +{ + struct asahi_g17_scope *scope = to_scope(fence); + if (atomic_read_acquire(&scope->pending)) + return false; + cmpxchg(&fence->error, 1, 0); + return true; +} + +static bool scope_enable_signaling(struct dma_fence *fence) +{ + /* Child callbacks are already armed; no recursive callback installation. */ + return !scope_signaled(fence); +} + +unsigned int asahi_g17_scope_children(struct dma_fence *fence, + struct dma_fence *children[2]) +{ + struct asahi_g17_scope *scope = to_scope(fence); + unsigned long flags; + unsigned int i, count = 0; + + spin_lock_irqsave(&fence->inline_lock, flags); + if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) + if (scope->children[i]) + children[count++] = dma_fence_get(scope->children[i]); + spin_unlock_irqrestore(&fence->inline_lock, flags); + return count; +} + +static void scope_release_completed(struct asahi_g17_scope *scope) +{ + struct dma_fence *children[2]; + unsigned long flags; + unsigned int i; + + /* Readers take references under this lock. Signalled scopes need no history. */ + spin_lock_irqsave(&scope->base.inline_lock, flags); + for (i = 0; i < ARRAY_SIZE(children); ++i) { + children[i] = scope->children[i]; + scope->children[i] = NULL; + } + spin_unlock_irqrestore(&scope->base.inline_lock, flags); + for (i = 0; i < ARRAY_SIZE(children); ++i) + dma_fence_put(children[i]); + dma_fence_put(&scope->base); +} + +static void scope_complete(struct asahi_g17_scope *scope) +{ + cmpxchg(&scope->base.error, 1, 0); + dma_fence_signal(&scope->base); + scope_release_completed(scope); +} + +static void scope_work(struct irq_work *work) +{ + struct asahi_g17_scope *scope = container_of(work, struct asahi_g17_scope, work); + cmpxchg(&scope->base.error, 1, 0); + asahi_g17_signal_fence(&scope->base); + scope_release_completed(scope); +} + +static void scope_defer_list(struct llist_node *nodes) +{ + struct llist_node *node, *next; + llist_for_each_safe(node, next, nodes) { + struct asahi_g17_scope *scope = + container_of(node, struct asahi_g17_scope, ready); + /* Transfer the ready-list reference to the irq_work callback. */ + irq_work_queue(&scope->work); + } +} + +void asahi_g17_signal_fence(struct dma_fence *fence) +{ + struct llist_head ready = LLIST_HEAD_INIT(ready); + struct llist_node *nodes, *node, *next; + unsigned long flags; + unsigned int completed = 0; + + /* Fence callbacks are atomic. Keep the local ready list pinned while + * signaling, then finish ready scopes after the source fence is unlocked. + * New ready scopes append to this list instead of recursing or sending IPIs. + */ + preempt_disable(); + if (__this_cpu_read(active_scopes)) { + dma_fence_signal(fence); + preempt_enable(); + return; + } + __this_cpu_write(active_scopes, &ready); + dma_fence_signal(fence); + for (;;) { + /* An interrupt must not append after the final empty test. */ + local_irq_save(flags); + nodes = llist_del_all(&ready); + if (!nodes) + __this_cpu_write(active_scopes, NULL); + local_irq_restore(flags); + if (!nodes) + break; + llist_for_each_safe(node, next, nodes) { + if (completed == SCOPE_ATOMIC_BATCH) { + struct llist_node *more; + local_irq_save(flags); + more = llist_del_all(&ready); + __this_cpu_write(active_scopes, NULL); + local_irq_restore(flags); + preempt_enable(); + scope_defer_list(node); + scope_defer_list(more); + return; + } + scope_complete(container_of(node, struct asahi_g17_scope, ready)); + ++completed; + } + } + preempt_enable(); +} + +static void scope_arrived(struct asahi_g17_scope *scope, int error) +{ + if (error < 0) + cmpxchg(&scope->base.error, 1, error); + if (atomic_dec_and_test(&scope->pending)) { + struct llist_head *ready; + dma_fence_get(&scope->base); + preempt_disable(); + ready = __this_cpu_read(active_scopes); + if (ready) + llist_add(&scope->ready, ready); + preempt_enable(); + /* Foreign fence signalers have no driver completion pass to drain us. */ + if (!ready) + irq_work_queue(&scope->work); + } +} + +static void scope_callback(struct dma_fence *child, struct dma_fence_cb *callback) +{ + struct asahi_g17_scope_callback *cb = + container_of(callback, typeof(*cb), callback); + scope_arrived(cb->scope, child->error); + dma_fence_put(&cb->scope->base); +} + +static void scope_release(struct dma_fence *fence) +{ + struct asahi_g17_scope *scope = to_scope(fence); + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) + dma_fence_put(scope->children[i]); + dma_fence_free(fence); +} + +static void scope_set_deadline(struct dma_fence *fence, ktime_t deadline) +{ + struct asahi_g17_fence_leaf *leaves; + int count, i; + + if (dma_fence_is_signaled(fence)) + return; + /* A deadline is an advisory hint. Keep its traversal bounded and iterative, + * including when a foreign timeline boundary contains earlier scopes. + */ + leaves = kmalloc_array(128, sizeof(*leaves), GFP_ATOMIC | __GFP_NOWARN); + if (!leaves) + return; + count = asahi_g17_collect_fences(fence, leaves, 128); + for (i = 0; i < count; ++i) { + if (!asahi_g17_is_scope(leaves[i].fence)) + dma_fence_set_deadline(leaves[i].fence, deadline); + dma_fence_put(leaves[i].fence); + } + kfree(leaves); +} + +static const struct dma_fence_ops scope_ops = { + .get_driver_name = scope_driver_name, + .get_timeline_name = scope_timeline_name, + .enable_signaling = scope_enable_signaling, + .signaled = scope_signaled, + .release = scope_release, + .set_deadline = scope_set_deadline, +}; + +void asahi_g17_scope_bind(struct dma_fence *fence, struct dma_fence *root) +{ + struct asahi_g17_scope *scope = to_scope(fence); + /* A concurrent raw syncobj reset is invalid for the private queue anchor, + * but must not turn a hostile userspace race into a kernel dereference. + */ + if (!root || dma_fence_chain_contained(root) != fence) + return; + /* Written once before the submit returns; an unmatched snapshot takes fallback. */ + WRITE_ONCE(scope->anchor_point, root->seqno); + WRITE_ONCE(scope->anchor_context, root->context); +} + +struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, + struct dma_fence *submitted, u64 point) +{ + struct dma_fence *contained = previous ? dma_fence_chain_contained(previous) : NULL; + struct dma_fence *parent = NULL; + struct asahi_g17_scope *scope, *prior = NULL; + u64 context; + unsigned int i; + + if (asahi_g17_is_scope(contained)) { + prior = to_scope(contained); + if (READ_ONCE(prior->anchor_context) != previous->context || + READ_ONCE(prior->anchor_point) != previous->seqno || + point <= contained->seqno) + prior = NULL; + } + if (prior) { + parent = dma_fence_get(contained); + context = contained->context; + } else { + /* Empty submissions or external edits may insert a foreign timeline point. + * Flatten that boundary once; the ordinary path retains just the prior scope. + */ + if (previous) { + parent = dma_fence_unwrap_merge(previous); + if (!parent) + return NULL; + } + context = dma_fence_context_alloc(1); + } + scope = kzalloc_obj(*scope); + if (!scope) { + dma_fence_put(parent); + return NULL; + } + dma_fence_init64(&scope->base, &scope_ops, NULL, context, point); + init_irq_work(&scope->work, scope_work); + scope->children[0] = parent; + scope->children[1] = dma_fence_get(submitted); + /* The constructor guard prevents completion while callbacks are being installed. */ + atomic_set(&scope->pending, 3); + scope->base.error = 1; + for (i = 0; i < ARRAY_SIZE(scope->children); ++i) { + struct dma_fence *child = scope->children[i]; + if (!child) { + scope_arrived(scope, 0); + continue; + } + scope->callbacks[i].scope = scope; + dma_fence_get(&scope->base); + if (dma_fence_add_callback(child, &scope->callbacks[i].callback, scope_callback)) { + scope_arrived(scope, child->error); + dma_fence_put(&scope->base); + } + } + scope_arrived(scope, 0); + return &scope->base; +} diff --git a/drivers/gpu/drm/asahi/g17/sync_scope.h b/drivers/gpu/drm/asahi/g17/sync_scope.h new file mode 100644 index 0000000000000..e88b78aacdd7b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/sync_scope.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef ASAHI_G17_SYNC_SCOPE_H +#define ASAHI_G17_SYNC_SCOPE_H +#include +struct dma_fence; +void asahi_g17_signal_fence(struct dma_fence *fence); +/* Only for explicitly signalled driver completion fences, with no poll op. */ +int asahi_g17_completion_fence_status(struct dma_fence *fence); + +struct dma_fence *asahi_g17_scope_create(struct dma_fence *previous, + struct dma_fence *submitted, u64 point); +void asahi_g17_scope_bind(struct dma_fence *scope, struct dma_fence *root); +bool asahi_g17_is_scope(struct dma_fence *fence); +/* Returns owned child references, or zero once the scope is signalled. */ +unsigned int asahi_g17_scope_children(struct dma_fence *fence, + struct dma_fence *children[2]); +#endif diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs index d951bd164d6e9..7b4ef4137b795 100644 --- a/drivers/gpu/drm/asahi/g17/timeout.rs +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -53,6 +53,32 @@ pub(crate) fn renew_execution(published: bool, enabled: bool, renewals: &mut u8) true } +/// Polls a published job may be renewed while the GPU keeps retiring work. A job +/// shares the GPU's cores with every other runnable queue, so its wall time grows +/// with the other clients' work: beside four other clients a 1.5 s dispatch took up +/// to 5 s, and lower-priority work waits behind higher-priority work. It has not +/// hung while the GPU keeps completing commands. Fourteen renewals bound the wait at +/// thirty seconds, after which a job that still has not finished fails as before. +const SHARED_RENEWALS: u8 = 14; + +/// The GPU counts as live while some command retired this recently. Under heavy +/// sharing the longest gap between retirements measured 1.2 s; a GPU that retires +/// nothing for this long has stopped. +pub(crate) const LIVENESS_NS: u64 = 5_000_000_000; + +/// Renews a published job's deadline while some command retired within +/// [`LIVENESS_NS`] (`since_retired`), at most [`SHARED_RENEWALS`] times. +pub(crate) fn renew_shared(published: bool, since_retired: Option, renewals: &mut u8) -> bool { + if !published + || !since_retired.is_some_and(|since| since < LIVENESS_NS) + || *renewals >= SHARED_RENEWALS + { + return false; + } + *renewals += 1; + true +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct Head { /// The owning VM's [`crate::mmu::Vm::id`], one per VM status; the watchdog lines print it. @@ -215,6 +241,24 @@ mod tests { assert_eq!(POLL_MS * (1 + u32::from(ray) + u32::from(MAX_RENEWALS)), 26_000); } #[test] + fn a_live_gpu_renews_a_published_job_for_thirty_seconds() { + let mut renewals = 0; + for poll in 1..=14u8 { + assert!(renew_shared(true, Some(4_900_000_000), &mut renewals)); + assert_eq!(renewals, poll); + } + assert!(!renew_shared(true, Some(0), &mut renewals)); + assert_eq!(POLL_MS * (u32::from(renewals) + 1), 30_000); + } + #[test] + fn a_stopped_gpu_or_unpublished_job_keeps_the_deadline() { + let mut renewals = 0; + assert!(!renew_shared(true, Some(LIVENESS_NS), &mut renewals)); + assert!(!renew_shared(true, None, &mut renewals)); + assert!(!renew_shared(false, Some(0), &mut renewals)); + assert_eq!(renewals, 0); + } + #[test] fn exhausted_or_invalid_credit_never_wraps() { for value in 4..=u8::MAX { let mut renewals = value; diff --git a/drivers/gpu/drm/asahi/g17/trace.c b/drivers/gpu/drm/asahi/g17/trace.c new file mode 100644 index 0000000000000..23666be389b96 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include + +#include "trace_api.h" + +#define CREATE_TRACE_POINTS +#include "trace.h" + +bool asahi_g17_trace_publish_enabled(void) +{ + return trace_asahi_g17_publish_enabled(); +} + +void asahi_g17_trace_publish(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + struct dma_fence *fence) +{ + trace_asahi_g17_publish(engine, qid, ordinal, ctx, seq, + fence ? fence->context : 0, fence ? fence->seqno : 0, + __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_complete(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end) +{ + if (trace_asahi_g17_complete_enabled()) + trace_asahi_g17_complete(engine, qid, ordinal, ctx, seq, gpu_start, + gpu_end, __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind) +{ + trace_asahi_g17_bind(engine, qid, ctx, kind); +} + +bool asahi_g17_trace_doorbell_enabled(void) +{ + return trace_asahi_g17_doorbell_enabled(); +} + +void asahi_g17_trace_doorbell(u8 engine, u32 power) +{ + trace_asahi_g17_doorbell(engine, power, __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_message(u8 kind, u64 value) +{ + if (trace_asahi_g17_message_enabled()) + trace_asahi_g17_message(kind, value, __arch_counter_get_cntvct()); +} + +void asahi_g17_trace_event(u8 phase) +{ + if (trace_asahi_g17_event_enabled()) + trace_asahi_g17_event(phase, __arch_counter_get_cntvct()); +} diff --git a/drivers/gpu/drm/asahi/g17/trace.h b/drivers/gpu/drm/asahi/g17/trace.h new file mode 100644 index 0000000000000..7e5a50e774b59 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.h @@ -0,0 +1,148 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Command lifecycle tracepoints of the G17 runtime. Times are joined by + * (ctx, seq); `counter` is the CPU's virtual counter and `gpu_start`/`gpu_end` + * come from the firmware's completion record, so a reader can place firmware + * dispatch and completion on the host timeline. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM asahi_g17 + +#if !defined(ASAHI_G17_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define ASAHI_G17_TRACE_H + +#include + +TRACE_EVENT(asahi_g17_publish, + TP_PROTO(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 fence_context, u64 fence_seqno, u64 counter), + TP_ARGS(engine, qid, ordinal, ctx, seq, fence_context, fence_seqno, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ordinal) + __field(u32, ctx) + __field(u64, seq) + __field(u64, fence_context) + __field(u64, fence_seqno) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ordinal = ordinal; + __entry->ctx = ctx; + __entry->seq = seq; + __entry->fence_context = fence_context; + __entry->fence_seqno = fence_seqno; + __entry->counter = counter; + ), + TP_printk("engine=%u qid=%u ordinal=%u ctx=%u seq=%llu fence=%llu:%llu counter=%llu", + __entry->engine, __entry->qid, __entry->ordinal, __entry->ctx, + __entry->seq, __entry->fence_context, __entry->fence_seqno, + __entry->counter) +); + +TRACE_EVENT(asahi_g17_complete, + TP_PROTO(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end, u64 counter), + TP_ARGS(engine, qid, ordinal, ctx, seq, gpu_start, gpu_end, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ordinal) + __field(u32, ctx) + __field(u64, seq) + __field(u64, gpu_start) + __field(u64, gpu_end) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ordinal = ordinal; + __entry->ctx = ctx; + __entry->seq = seq; + __entry->gpu_start = gpu_start; + __entry->gpu_end = gpu_end; + __entry->counter = counter; + ), + TP_printk("engine=%u qid=%u ordinal=%u ctx=%u seq=%llu gpu=%llu..%llu counter=%llu", + __entry->engine, __entry->qid, __entry->ordinal, __entry->ctx, + __entry->seq, __entry->gpu_start, __entry->gpu_end, __entry->counter) +); + +TRACE_EVENT(asahi_g17_bind, + TP_PROTO(u8 engine, u8 qid, u32 ctx, u8 kind), + TP_ARGS(engine, qid, ctx, kind), + TP_STRUCT__entry( + __field(u8, engine) + __field(u8, qid) + __field(u32, ctx) + __field(u8, kind) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->qid = qid; + __entry->ctx = ctx; + __entry->kind = kind; + ), + TP_printk("engine=%u qid=%u ctx=%u kind=%u", + __entry->engine, __entry->qid, __entry->ctx, __entry->kind) +); + +TRACE_EVENT(asahi_g17_doorbell, + TP_PROTO(u8 engine, u32 power, u64 counter), + TP_ARGS(engine, power, counter), + TP_STRUCT__entry( + __field(u8, engine) + __field(u32, power) + __field(u64, counter) + ), + TP_fast_assign( + __entry->engine = engine; + __entry->power = power; + __entry->counter = counter; + ), + TP_printk("engine=%u power=%#x counter=%llu", __entry->engine, __entry->power, + __entry->counter) +); + +TRACE_EVENT(asahi_g17_message, + TP_PROTO(u8 kind, u64 value, u64 counter), + TP_ARGS(kind, value, counter), + TP_STRUCT__entry( + __field(u8, kind) + __field(u64, value) + __field(u64, counter) + ), + TP_fast_assign( + __entry->kind = kind; + __entry->value = value; + __entry->counter = counter; + ), + TP_printk("kind=%u value=%#llx counter=%llu", __entry->kind, __entry->value, + __entry->counter) +); + +TRACE_EVENT(asahi_g17_event, + TP_PROTO(u8 phase, u64 counter), + TP_ARGS(phase, counter), + TP_STRUCT__entry( + __field(u8, phase) + __field(u64, counter) + ), + TP_fast_assign( + __entry->phase = phase; + __entry->counter = counter; + ), + TP_printk("phase=%u counter=%llu", __entry->phase, __entry->counter) +); + +#endif + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace +#include diff --git a/drivers/gpu/drm/asahi/g17/trace.rs b/drivers/gpu/drm/asahi/g17/trace.rs new file mode 100644 index 0000000000000..b76b46425d2ce --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Command lifecycle tracepoints (tracefs `events/asahi_g17`). Each call is a +//! static-key test in C while its event is disabled. + +use kernel::{bindings, dma_fence::RawDmaFence, sync::Arc}; + +unsafe extern "C" { + fn asahi_g17_trace_publish_enabled() -> bool; + fn asahi_g17_trace_publish(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, + fence: *mut bindings::dma_fence); + fn asahi_g17_trace_complete(engine: u8, qid: u8, ordinal: u32, ctx: u32, seq: u64, + gpu_start: u64, gpu_end: u64); + fn asahi_g17_trace_bind(engine: u8, qid: u8, ctx: u32, kind: u8); + fn asahi_g17_trace_message(kind: u8, value: u64); + fn asahi_g17_trace_doorbell_enabled() -> bool; + fn asahi_g17_trace_doorbell(engine: u8, power: u32); + fn asahi_g17_trace_event(phase: u8); +} + +pub(super) const RENDER: u8 = 0; +pub(super) const COMPUTE: u8 = 1; + +/// Called once the command's doorbell has been rung. +pub(super) fn publish(engine: u8, qid: u8, ordinal: u32, packet: &Arc) { + // SAFETY: Pure static-key query. + if !unsafe { asahi_g17_trace_publish_enabled() } { + return; + } + let fence = packet.completion.fence(); + // SAFETY: `fence` holds a reference for the duration of the call. + unsafe { + asahi_g17_trace_publish(engine, qid, ordinal, packet.context.id(), packet.order.sequence, + fence.raw()) + }; +} + +/// Called when exact completion with GPU timestamps is retired. +pub(super) fn complete(engine: u8, qid: u8, ordinal: u32, packet: &super::job::Packet, + gpu: [u64; 2]) +{ + // SAFETY: Plain values only. + unsafe { + asahi_g17_trace_complete(engine, qid, ordinal, packet.context.id(), packet.order.sequence, + gpu[0], gpu[1]) + }; +} + +/// A physical queue was bound to a logical owner: 0 fresh, 1 rebound, 2 same VM. +pub(super) fn bind(engine: u8, qid: u8, ctx: u32, kind: u8) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_bind(engine, qid, ctx, kind) }; +} + +/// A work doorbell is about to ring; `power` samples the core power state +/// only while the event is enabled. +pub(super) fn doorbell(engine: u8, power: impl FnOnce() -> u32) { + // SAFETY: Pure static-key query. + if !unsafe { asahi_g17_trace_doorbell_enabled() } { + return; + } + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_doorbell(engine, power()) }; +} + +pub(super) const MAILBOX: u8 = 0; +pub(super) const CONTROL: u8 = 1; +pub(super) const EFFORT: u8 = 2; + +/// A host-to-firmware message: a mailbox message, a device-control record by +/// opcode, or a new performance effort value. +pub(super) fn message(kind: u8, value: u64) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_message(kind, value) }; +} + +/// Event worker pass boundaries: 0 begin, 1 fences signalled, 2 end. +pub(super) fn event(phase: u8) { + // SAFETY: Plain values only. + unsafe { asahi_g17_trace_event(phase) }; +} diff --git a/drivers/gpu/drm/asahi/g17/trace_api.h b/drivers/gpu/drm/asahi/g17/trace_api.h new file mode 100644 index 0000000000000..d09f529a163f0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/trace_api.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Entry points of the G17 command lifecycle tracepoints, called from Rust + * (g17/trace.rs). + */ +#ifndef _ASAHI_G17_TRACE_API_H +#define _ASAHI_G17_TRACE_API_H + +#include + +struct dma_fence; + +bool asahi_g17_trace_publish_enabled(void); +void asahi_g17_trace_publish(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + struct dma_fence *fence); +void asahi_g17_trace_complete(u8 engine, u8 qid, u32 ordinal, u32 ctx, u64 seq, + u64 gpu_start, u64 gpu_end); +void asahi_g17_trace_bind(u8 engine, u8 qid, u32 ctx, u8 kind); +bool asahi_g17_trace_doorbell_enabled(void); +void asahi_g17_trace_doorbell(u8 engine, u32 power); +void asahi_g17_trace_message(u8 kind, u64 value); +void asahi_g17_trace_event(u8 phase); + +#endif /* _ASAHI_G17_TRACE_API_H */ diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs index b2a6cf6e8bfe3..b0f2861382ad7 100644 --- a/drivers/gpu/drm/asahi/g17/validation.rs +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -8,19 +8,41 @@ use core::{cell::RefCell, ops::Range}; use kernel::prelude::*; const MAX_RANGES: usize = 48; +// The persistent proof must retain a deep queue's rotating command buffers. +// Keep the transient parser request set small to bound its stack frame. +const MAX_CACHED_RANGES: usize = 128; type Request = (u64, u64, u8); +/// Match the page footprint already checked by `Vm::covers_range_locked`. +fn pages((address, size, access): Request) -> Option { + if size == 0 || access == 0 || access & !3 != 0 { + return None; + } + let mask = mmu::UAT_PGSZ as u64 - 1; + let end = address.checked_add(size)?; + let end = end.checked_add(mask)? & !mask; + let start = address & !mask; + Some((start, end.checked_sub(start)?, access)) +} + +fn contains((start, size, access): Request, (other, length, needed): Request) -> bool { + access & needed == needed + && other >= start + && other.checked_add(length).is_some_and(|end| + start.checked_add(size).is_some_and(|limit| end <= limit)) +} + #[derive(Clone, Copy)] -struct Requests { - ranges: [Request; MAX_RANGES], +struct Requests { + ranges: [Request; CAPACITY], count: usize, overflow: bool, } -impl Requests { +impl Requests { const fn new() -> Self { Self { - ranges: [(0, 0, 0); MAX_RANGES], + ranges: [(0, 0, 0); CAPACITY], count: 0, overflow: false, } @@ -30,24 +52,55 @@ impl Requests { &self.ranges[..self.count] } + fn covers(&self, request: Request) -> bool { + self.as_slice().iter().any(|&range| contains(range, request)) + } + fn insert(&mut self, request: Request) { - if self.as_slice().contains(&request) { + let Some(mut request) = pages(request) else { + self.overflow = true; + return; + }; + if self.covers(request) { return; } - if self.count == MAX_RANGES { + // Merge only equal permissions, so an adjacent writable page never + // grants write access to a read-only page. Restart after each merge + // because it can join a range visited earlier. + let mut index = 0; + while index < self.count { + let range = self.ranges[index]; + let end = request.0 + request.1; + let other_end = range.0 + range.1; + if contains(request, range) + || (request.2 == range.2 && request.0 <= other_end && range.0 <= end) + { + if request.2 == range.2 { + let start = request.0.min(range.0); + request = (start, end.max(other_end) - start, request.2); + } + self.count -= 1; + self.ranges[index] = self.ranges[self.count]; + index = 0; + } else { + index += 1; + } + } + if self.count == CAPACITY { self.overflow = true; return; } self.ranges[self.count] = request; self.count += 1; } + } -/// One logical queue's bounded cache of exact address, size and permission tuples. +/// One logical queue's bounded cache of validated page ranges and permissions. /// Mapping mutations make the epoch unavailable until their final page-table write. pub(crate) struct Proof { epoch: Option, - requests: Requests, + requests: Requests, } impl Proof { @@ -64,7 +117,7 @@ impl Proof { && requests .as_slice() .iter() - .all(|request| self.requests.as_slice().contains(request)) + .all(|&request| self.requests.covers(request)) } fn record(&mut self, epoch: Option, requests: Requests) { @@ -79,7 +132,10 @@ impl Proof { } // An incomplete union cannot be reused; retain the latest complete set. self.epoch = if requests.overflow { None } else { epoch }; - self.requests = requests; + self.requests = Requests::new(); + for &request in requests.as_slice() { + self.requests.insert(request); + } } /// The VM-job guard must already pin the mappings for this command. Keep the @@ -144,3 +200,101 @@ impl AddressSpace for Space<'_> { } } } + + +#[kunit_tests(asahi_g17_validation_pages)] +mod tests { + use super::*; + const PAGE: u64 = mmu::UAT_PGSZ as u64; + + fn requests(range: Request) -> Requests { + let mut result = Requests::new(); + result.insert(range); + result + } + + #[test] + fn different_offsets_share_the_already_validated_page() { + let mut proof = Proof::new(); + proof.record(Some(7), requests((PAGE + 4, 4, 3))); + assert!(proof.covers(Some(7), &requests((PAGE + 0x100, 8, 1)))); + assert!(proof.covers(Some(7), &requests((2 * PAGE - 8, 8, 2)))); + assert!(!proof.covers(Some(7), &requests((2 * PAGE - 8, 16, 1)))); + assert!(!proof.covers(Some(8), &requests((PAGE + 4, 4, 1)))); + assert!(!proof.covers(None, &requests((PAGE + 4, 4, 1)))); + } + + #[test] + fn adjacent_equal_permissions_merge_but_holes_do_not() { + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((3 * PAGE, 1, 1)); + assert_eq!(ranges.count, 2); + assert!(!ranges.covers((2 * PAGE, PAGE, 1))); + ranges.insert((2 * PAGE, 1, 1)); + assert_eq!(ranges.count, 1); + assert_eq!(ranges.as_slice(), &[(PAGE, 3 * PAGE, 1)]); + } + + #[test] + fn adjacent_permissions_never_grant_access_to_another_page() { + let mut proof = Proof::new(); + let mut ranges = requests((PAGE, 1, 1)); + ranges.insert((2 * PAGE, 1, 3)); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((PAGE, 1, 2)))); + assert!(proof.covers(Some(1), &requests((2 * PAGE, 1, 2)))); + assert!(!proof.covers(Some(1), &requests((PAGE, 2 * PAGE, 3)))); + } + + #[test] + fn stronger_coverage_removes_only_fully_covered_ranges() { + let mut ranges = requests((PAGE, PAGE, 1)); + ranges.insert((2 * PAGE, PAGE, 2)); + ranges.insert((PAGE, 2 * PAGE, 3)); + assert_eq!(ranges.as_slice(), &[(PAGE, 2 * PAGE, 3)]); + assert!(!ranges.covers((0, PAGE, 1))); + } + + #[test] + fn persistent_cache_retains_a_deep_rotating_working_set() { + let mut proof = Proof::new(); + for index in 0..64 { + proof.record(Some(7), requests((index * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 64); + for index in 0..64 { + assert!(proof.covers(Some(7), &requests((index * 2 * PAGE + 16, 8, 1)))); + } + assert_eq!(core::mem::size_of::(), core::mem::size_of::>()); + } + + #[test] + fn persistent_overflow_keeps_only_a_new_complete_proof() { + let mut proof = Proof::new(); + for index in 0..=MAX_CACHED_RANGES { + proof.record(Some(7), requests((index as u64 * 2 * PAGE, 8, 1))); + } + assert_eq!(proof.requests.count, 1); + assert!(!proof.covers(Some(7), &requests((0, 8, 1)))); + assert!(proof.covers(Some(7), + &requests((MAX_CACHED_RANGES as u64 * 2 * PAGE, 8, 1)))); + } + + #[test] + fn invalid_rounding_and_full_cache_disable_reuse() { + for range in [(0, 0, 1), (u64::MAX - 1, 1, 1), (0, 1, 0), (0, 1, 4)] { + assert!(pages(range).is_none()); + let mut proof = Proof::new(); + proof.record(Some(1), requests(range)); + assert!(!proof.covers(Some(1), &requests((0, 1, 1)))); + } + let mut ranges = Requests::new(); + for index in 0..=MAX_RANGES { + ranges.insert(((2 * index) as u64 * PAGE, PAGE, 1)); + } + assert!(ranges.overflow); + let mut proof = Proof::new(); + proof.record(Some(1), ranges); + assert!(!proof.covers(Some(1), &requests((0, PAGE, 1)))); + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 0a0a51a02cfba..3522010949e35 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1201,56 +1201,85 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } -/// Prepared CPU timestamp access; destruction happens after completion signalling. -pub(crate) struct TimestampMapping { - mapping: kernel::drm::gem::shmem::VMapOwned, - offset: usize, -} +/// A kernel CPU mapping of a timestamp buffer's GEM backing. Dropping it takes the object's +/// reservation lock, so it must not be dropped where fence producers run. +pub(crate) struct TimestampVmap(kernel::drm::gem::shmem::VMapOwned); -impl TimestampMapping { - /// Prepare one bounded CPU timestamp destination on retained GEM backing. - pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { +impl TimestampVmap { + pub(crate) fn new(gem: &gem::Object) -> Result { + Ok(Self(gem.owned_vmap::<0>()?)) + } + + fn size(&self) -> usize { use kernel::drm::gem::BaseObject; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - Ok(Self { - mapping: gem.owned_vmap::<0>()?, - offset, - }) + self.0.owner().size() } - pub(crate) fn write(&self, value: u64) { + /// Writes a timestamp at byte `offset` of the GEM object. Aligned results use a complete + /// atomic store; byte stores support unaligned destinations without creating mutable + /// aliases. An unaligned complete timestamp is not atomic. Out-of-range offsets are refused + /// when the destination is prepared, and ignored here. + pub(crate) fn write(&self, offset: usize, value: u64) { + if offset.checked_add(size_of::()).is_none_or(|end| end > self.size()) { + return; + } // Completion runs in task context. Bindings of the same BO share this lock, // so aligned u64 and overlapping unaligned byte stores never race in Rust. - let _guard = self.mapping.owner().timestamp_lock.lock(); - // SAFETY: timestamp_mapping checked the complete extent inside this pinned vmap. - let base = self.mapping.addr.cast::(); - let pointer = unsafe { base.add(self.offset) }; + let _guard = self.0.owner().timestamp_lock.lock(); + // SAFETY: The complete extent was checked above, inside this pinned vmap. + let base = self.0.addr.cast::(); + let pointer = unsafe { base.add(offset) }; if pointer as usize % core::mem::align_of::() == 0 { - // SAFETY: timestamp_mapping checked all eight bytes; the vmap pins the backing. + // SAFETY: All eight bytes were checked above; the vmap pins the backing. // A naturally aligned query result becomes available in one complete store. let destination = unsafe { &*pointer.cast::() }; destination.store(value.to_le(), Ordering::Relaxed); } else { for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { &*base.add(self.offset + index).cast::() }; + // SAFETY: The complete eight-byte extent was checked above. The owned + // vmap pins its backing; byte atomics permit unaligned and overlapping + // timestamp destinations without mutable slice aliases. + let destination = unsafe { &*base.add(offset + index).cast::() }; destination.store(byte, Ordering::Relaxed); } } // A CPU-retired query can be consumed by a later GPU command. WRITEBACK // objects use a cacheable vmap, while GPU mappings need not snoop that cache. // Publish the bytes before signalling the completion fence or queue frontier. - if self.mapping.owner().is_writeback() { + if self.0.owner().is_writeback() { crate::pgtable::clean_poc(pointer, size_of::()); } crate::pgtable::poc_barrier(); } } +/// Prepared CPU timestamp access with its own vmap; destruction happens after completion +/// signalling. +#[cfg(CONFIG_DRM_ASAHI_G16)] +pub(crate) struct TimestampMapping { + mapping: TimestampVmap, + offset: usize, +} + +#[cfg(CONFIG_DRM_ASAHI_G16)] +impl TimestampMapping { + /// Prepare one bounded CPU timestamp destination on retained GEM backing. + pub(crate) fn new(gem: &gem::Object, offset: usize) -> Result { + use kernel::drm::gem::BaseObject; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + Ok(Self { + mapping: TimestampVmap::new(gem)?, + offset, + }) + } + + pub(crate) fn write(&self, value: u64) { + self.mapping.write(self.offset, value); + } +} + /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. pub(crate) struct KernelMapping(ManuallyDrop>); @@ -1285,7 +1314,15 @@ impl KernelMapping { /// use a complete atomic store; byte stores support unaligned destinations without /// creating mutable aliases. An unaligned complete timestamp is not atomic. /// Retain CPU access before this destination's completion fence is published. + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + TimestampMapping::new(gem, self.timestamp_offset(offset)?) + } + + /// The GEM object offset of the timestamp at `offset` in this mapping, whose complete + /// extent lies inside both the mapping and the object. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1296,7 +1333,12 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - TimestampMapping::new(gem, start) + Ok(start) + } + + /// A kernel CPU mapping of this mapping's GEM backing, for timestamp writes. + pub(crate) fn timestamp_vmap(&self) -> Result { + TimestampVmap::new(self.0._gem.as_ref().ok_or(EINVAL)?) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index e7f6fd810b4ad..e8b845cba989f 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -709,6 +709,11 @@ impl G17Resources { self.sgx_read32::() & 0xf != 0 } + /// Raw core power state, for tracing only. + pub(crate) fn g17_core_power(&self) -> u32 { + self.sgx_read32::() + } + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is /// initialized. pub(crate) fn enable_g17_gating(&self) -> Result { diff --git a/drivers/gpu/drm/asahi/timestamp.rs b/drivers/gpu/drm/asahi/timestamp.rs index 976ec9f6bf48b..9f13436b4caf5 100644 --- a/drivers/gpu/drm/asahi/timestamp.rs +++ b/drivers/gpu/drm/asahi/timestamp.rs @@ -13,10 +13,20 @@ use kernel::{prelude::*, sync::Arc}; #[cfg(CONFIG_DRM_ASAHI_G16)] use kernel::{uapi, xarray}; +/// A firmware-mapped timestamp buffer with a kernel CPU mapping of its backing, kept for the +/// buffer's lifetime so that commands need no mapping of their own. +pub(crate) struct FirmwareBuffer { + mapping: Arc, + cpu: mmu::TimestampVmap, +} + /// A bound timestamp destination. Only backends which publish firmware writes need a UAT alias. /// CPU-retired timestamps retain their backing without consuming global firmware address space. pub(crate) enum Buffer { - Firmware(Arc), + /// The last reference may be a command's, dropped where fence producers run, while + /// releasing the CPU mapping takes the reservation lock: the deferral runs the release + /// on the cleanup queue. + Firmware(crate::cleanup::Deferred), #[cfg(CONFIG_DRM_ASAHI_G16)] Cpu { // The GEM pin keeps the object's held SG table alive through CPU publication. @@ -27,7 +37,11 @@ pub(crate) enum Buffer { impl Buffer { pub(crate) fn firmware(mapping: mmu::KernelMapping) -> Result { - Ok(Self::Firmware(Arc::new(mapping, GFP_KERNEL)?)) + let cpu = mapping.timestamp_vmap()?; + Ok(Self::Firmware(crate::cleanup::Deferred::new(FirmwareBuffer { + mapping: Arc::new(mapping, GFP_KERNEL)?, + cpu, + })?)) } /// Retain a CPU-written byte range without publishing any firmware address. @@ -46,15 +60,16 @@ impl Buffer { pub(crate) fn size(&self) -> usize { match self { - Self::Firmware(mapping) => mapping.size(), + Self::Firmware(buffer) => buffer.mapping.size(), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { range, .. } => range.end - range.start, } } + #[cfg(CONFIG_DRM_ASAHI_G16)] pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { match self { - Self::Firmware(mapping) => mapping.timestamp_mapping(offset), + Self::Firmware(buffer) => buffer.mapping.timestamp_mapping(offset), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { gem, range, .. } => { if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -65,10 +80,30 @@ impl Buffer { } } + /// The object offset of the timestamp at `offset` of a firmware buffer, for + /// [`Self::write_timestamp`]. + pub(crate) fn timestamp_offset(&self, offset: usize) -> Result { + match self { + Self::Firmware(buffer) => buffer.mapping.timestamp_offset(offset), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => Err(EINVAL), + } + } + + /// Writes a timestamp at an object offset returned by [`Self::timestamp_offset`], + /// through the buffer's own CPU mapping. + pub(crate) fn write_timestamp(&self, offset: usize, value: u64) { + match self { + Self::Firmware(buffer) => buffer.cpu.write(offset, value), + #[cfg(CONFIG_DRM_ASAHI_G16)] + Self::Cpu { .. } => {} + } + } + /// Legacy command encoders publish firmware pointers and must never accept a CPU-only buffer. pub(crate) fn firmware_mapping(&self) -> Result> { match self { - Self::Firmware(mapping) => Ok(mapping.clone()), + Self::Firmware(buffer) => Ok(buffer.mapping.clone()), #[cfg(CONFIG_DRM_ASAHI_G16)] Self::Cpu { .. } => Err(EINVAL), } diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 66396f89cc333..66d5188199383 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -368,6 +368,13 @@ enum drm_asahi_feature { * DRM_ASAHI_COMPUTE_AS_BUILD for a bounded long execution allowance. */ DRM_ASAHI_FEATURE_BOUNDED_AS_BUILD = (1UL) << 16, + + /** + * @DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS: SUBMIT accepts one + * DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE output. All outputs + * include the earlier points of that timeline without adding GPU waits. + */ + DRM_ASAHI_FEATURE_CUMULATIVE_SIGNALS = (1UL) << 29, }; #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 @@ -901,6 +908,16 @@ enum drm_asahi_sync_type { /** @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */ DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ = 1, + + /** + * @DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE: Output-only queue + * timeline anchor, requiring CUMULATIVE_SIGNALS and a nonzero point. + * At most one is allowed. All outputs receive completion of this submit + * and the earlier anchor points. + * Other outputs must not alias the anchor. No execution dependency is + * added. Calls updating this private anchor must be externally serialized. + */ + DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ_CUMULATIVE = 2, }; /** diff --git a/rust/kernel/drm/syncobj.rs b/rust/kernel/drm/syncobj.rs index a022e08223588..72ba50c57286b 100644 --- a/rust/kernel/drm/syncobj.rs +++ b/rust/kernel/drm/syncobj.rs @@ -15,6 +15,14 @@ pub struct SyncObj { } impl SyncObj { + /// Borrows the raw sync object for synchronous calls into DRM helpers. + /// The caller must not release the owned reference or retain this pointer + /// beyond the lifetime of `self` without taking another reference. + #[inline(always)] + pub fn as_raw(&self) -> *mut bindings::drm_syncobj { + self.ptr + } + /// Looks up a sync object by its handle for a given `File`. pub fn lookup_handle( file: &drm::File,