drm/asahi: improve G17 submission, queue admission and completion

drivers/dma-buf/dma-fence-array.c

@@ -107,17 +107,11 @@ static bool dma_fence_array_signaled(struct dma_fence *fence)
 	unsigned int i;
 
 	/*
-	 * We need to read num_pending before checking the enable_signal bit
-	 * to avoid racing with the enable_signaling() implementation, which
-	 * might decrement the counter, and cause a partial check.
-	 * atomic_read_acquire() pairs with atomic_dec_and_test() in
-	 * dma_fence_array_enable_signaling()
-	 *
-	 * The !--num_pending check is here to account for the any_signaled case
-	 * if we race with enable_signaling(), that means the !num_pending check
-	 * in the is_signalling_enabled branch might be outdated (num_pending
-	 * might have been decremented), but that's fine. The user will get the
-	 * right value when testing again later.
+	 * Read the counter before the enable bit so callback installation
+	 * cannot make a partial count look complete. The acquire pairs with
+	 * the counter decrements in dma_fence_array_enable_signaling().
+	 * If enabling races with the lazy scan, every child must still be
+	 * observed complete before that scan can return true.
 	 */
 	num_pending = atomic_read_acquire(&array->num_pending);
 	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &array->base.flags)) {
@@ -126,8 +120,16 @@ static bool dma_fence_array_signaled(struct dma_fence *fence)
 		return false;
 	}
 
+	/*
+	 * Arrays require every member to complete. A pending member already
+	 * proves the array incomplete; polling its successors only repeats work
+	 * during timeline-chain walks. Keep the completion counter and the
+	 * enabled-signaling path unchanged.
+	 */
 	for (i = 0; i < array->num_fences; ++i) {
-		if (dma_fence_is_signaled(array->fences[i]) && !--num_pending)
+		if (!dma_fence_is_signaled(array->fences[i]))
+			return false;
+		if (!--num_pending)
 			goto signal;
 	}
 	return false;
@@ -256,8 +258,7 @@ EXPORT_SYMBOL(dma_fence_array_init);
  * and fill it with the fences it wants to add to the object. Ownership of this
  * array is taken and dma_fence_put() is used on each fence on release.
  *
- * If @signal_on_any is true the fence array signals if any fence in the array
- * signals, otherwise it signals when all fences in the array signal.
+ * The array signals when every fence in the array signals.
  */
 struct dma_fence_array *dma_fence_array_create(int num_fences,
 					       struct dma_fence **fences,

drivers/dma-buf/st-dma-fence-unwrap.c

@@ -15,16 +15,26 @@
 struct mock_fence {
 	struct dma_fence base;
 	spinlock_t lock;
+	atomic_t polls;
 };
 
 static const char *mock_name(struct dma_fence *f)
 {
 	return "mock";
 }
 
+static bool mock_signaled(struct dma_fence *fence)
+{
+	struct mock_fence *f = container_of(fence, struct mock_fence, base);
+
+	atomic_inc(&f->polls);
+	return false;
+}
+
 static const struct dma_fence_ops mock_ops = {
 	.get_driver_name = mock_name,
 	.get_timeline_name = mock_name,
+	.signaled = mock_signaled,
 };
 
 static struct dma_fence *__mock_fence(u64 context, u64 seqno)
@@ -36,6 +46,7 @@ static struct dma_fence *__mock_fence(u64 context, u64 seqno)
 		return NULL;
 
 	spin_lock_init(&f->lock);
+	atomic_set(&f->polls, 0);
 	dma_fence_init(&f->base, &mock_ops, &f->lock, context, seqno);
 
 	return &f->base;
@@ -584,7 +595,71 @@ static void test_unwrap_merge_complex_seqno(struct kunit *test)
 	dma_fence_put(f1);
 }
 
+/*
+ * A pending all-of array needs no polls of later members. Keep its
+ * independent member fences so their polling and out-of-order completion
+ * remain observable without enabling callbacks on the array.
+ */
+static void test_array_pending_prefix(struct kunit *test)
+{
+	struct dma_fence *first = mock_fence(), *last = mock_fence();
+	struct dma_fence *array;
+	struct mock_fence *last_mock;
+
+	if (!first || !last) {
+		KUNIT_FAIL(test, "Could not allocate member fences");
+		goto out;
+	}
+	last_mock = container_of(last, struct mock_fence, base);
+	array = mock_array(2, dma_fence_get(first), dma_fence_get(last));
+	if (!array) {
+		KUNIT_FAIL(test, "Could not allocate fence array");
+		goto out;
+	}
+	KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array));
+	KUNIT_EXPECT_EQ(test, atomic_read(&last_mock->polls), 0);
+	dma_fence_signal(last);
+	KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array));
+	dma_fence_signal(first);
+	KUNIT_EXPECT_TRUE(test, dma_fence_is_signaled(array));
+	dma_fence_put(array);
+out:
+	dma_fence_put(last);
+	dma_fence_put(first);
+}
+
+static void test_array_pending_suffix(struct kunit *test)
+{
+	struct dma_fence *first = mock_fence(), *last = mock_fence();
+	struct dma_fence *array;
+
+	if (!first || !last) {
+		KUNIT_FAIL(test, "Could not allocate member fences");
+		goto out;
+	}
+	array = mock_array(2, dma_fence_get(first), dma_fence_get(last));
+	if (!array) {
+		KUNIT_FAIL(test, "Could not allocate fence array");
+		goto out;
+	}
+	dma_fence_signal(first);
+	KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array));
+	/* Installing callbacks after a lazy partial poll must still account
+	 * for the already signaled first member and wait for the last.
+	 */
+	dma_fence_enable_signaling(array);
+	KUNIT_EXPECT_FALSE(test, dma_fence_is_signaled(array));
+	dma_fence_signal(last);
+	KUNIT_EXPECT_GT(test, dma_fence_wait_timeout(array, false, HZ), 0L);
+	dma_fence_put(array);
+out:
+	dma_fence_put(last);
+	dma_fence_put(first);
+}
+
 static struct kunit_case dma_fence_unwrap_cases[] = {
+	KUNIT_CASE(test_array_pending_prefix),
+	KUNIT_CASE(test_array_pending_suffix),
 	KUNIT_CASE(test_sanitycheck),
 	KUNIT_CASE(test_unwrap_array),
 	KUNIT_CASE(test_unwrap_chain),

drivers/gpu/drm/asahi/Makefile

@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 
 obj-$(CONFIG_DRM_ASAHI) += asahi.o
-asahi-y := asahi_core.o fwid.o sysfs.o
+asahi-y := asahi_core.o fwid.o sysfs.o g17/sync_input.o g17/sync_output.o g17/sync_scope.o g17/trace.o
+CFLAGS_g17/trace.o := -I$(src)/g17
 
 asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o
 

drivers/gpu/drm/asahi/file.rs

@@ -127,10 +127,18 @@ pub(crate) struct SyncItem {
     pub(crate) fence: Option<dma_fence::Fence>,
     pub(crate) chain_fence: Option<dma_fence::FenceChain>,
     pub(crate) timeline_value: u64,
+    pub(crate) cumulative: Option<crate::g17::sync_output::SignalBundle>,
+    pub(crate) timeline: bool,
 }
 
 impl SyncItem {
-    fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result<SyncItem> {
+    fn parse_one(file: &DrmFile, mut data: uapi::drm_asahi_sync, out: bool,
+        bundled: bool) -> Result<SyncItem>
+    {
+        if data.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE {
+            if !out || !bundled || data.timeline_value == 0 { return Err(EINVAL); }
+            data.sync_type = uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ;
+        }
         match data.sync_type {
             uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => {
                 if data.timeline_value != 0 {
@@ -151,6 +159,8 @@ impl SyncItem {
                     syncobj,
                     chain_fence: None,
                     timeline_value: data.timeline_value,
+                    cumulative: None,
+                    timeline: false,
                 })
             }
             uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => {
@@ -173,12 +183,10 @@ impl SyncItem {
                 Ok(SyncItem {
                     fence,
                     syncobj,
-                    chain_fence: if out {
-                        Some(dma_fence::FenceChain::new()?)
-                    } else {
-                        None
-                    },
+                    chain_fence: None,
                     timeline_value: data.timeline_value,
+                    cumulative: None,
+                    timeline: true,
                 })
             }
             _ => {
@@ -193,12 +201,14 @@ impl SyncItem {
         ptr: u64,
         in_count: u32,
         out_count: u32,
+        allow_cumulative: bool,
     ) -> Result<KVec<SyncItem>> {
         let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?;
 
         const STRIDE: usize = core::mem::size_of::<uapi::drm_asahi_sync>();
         let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?;
         let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?;
+        let mut anchor = None;
 
         // SAFETY: We only read this once, so there are no TOCTOU issues.
         let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader();
@@ -214,7 +224,25 @@ impl SyncItem {
             // SAFETY: All bit patterns in the struct are valid
             let sync = unsafe { sync.assume_init() };
 
-            vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?;
+            if sync.sync_type == crate::g17::sync_output::SYNC_CUMULATIVE {
+                if !allow_cumulative || i < in_count || anchor.is_some() {
+                    return Err(EINVAL);
+                }
+                anchor = Some(i as usize);
+            }
+            vec.push(Self::parse_one(file, sync, i >= in_count,
+                allow_cumulative)?, GFP_KERNEL)?;
+        }
+        if let Some(anchor) = anchor {
+            let bundle = crate::g17::sync_output::SignalBundle::new(
+                &vec[in_count as usize..], anchor - in_count as usize)?;
+            vec[anchor].cumulative = Some(bundle);
+        } else {
+            for sync in &mut vec[in_count as usize..] {
+                if sync.timeline {
+                    sync.chain_fence = Some(dma_fence::FenceChain::new()?);
+                }
+            }
         }
 
         Ok(vec)
@@ -1398,7 +1426,8 @@ impl File {
             copy_commands(&mut vec)?;
         }
         let syncs =
-            SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?;
+            SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count,
+                gpu.as_any().is::<crate::g17::Gpu>())?;
 
         mod_dev_dbg!(
             device,

drivers/gpu/drm/asahi/g17/availability.rs

@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+
+//! Resource-specific scheduler waits. A caller samples a domain before a failed try, then
+//! registers against that sample after rolling back unpublished claims; the release that frees
+//! the resource bumps the generation and signals exactly the waiters of its own domain.
+//! Successful acquisition never allocates a fence, and a release without waiters costs one
+//! lock round trip.
+
+use super::fence::{self, CompletionFence};
+use kernel::{
+    c_str,
+    dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence},
+    new_mutex,
+    prelude::*,
+    sync::{Arc, Mutex},
+};
+
+struct Entry {
+    fence: UserFence<CompletionFence>,
+    next: Option<KBox<Entry>>,
+}
+
+struct State {
+    generation: u64,
+    waiters: Option<KBox<Entry>>,
+}
+
+#[pin_data]
+pub(crate) struct Availability {
+    #[pin]
+    state: Mutex<State>,
+    fences: FenceContexts,
+}
+
+impl Availability {
+    pub(crate) fn new() -> Result<Arc<Self>> {
+        Arc::pin_init(
+            try_pin_init!(Self {
+                state <- new_mutex!(
+                    State { generation: 0, waiters: None },
+                    "g17::Availability"
+                ),
+                fences: FenceContexts::new(1, c_str!("asahi_resource"), kernel::static_lock_class!())?,
+            }),
+            GFP_KERNEL,
+        )
+    }
+
+    /// Sample before checking capacity under the resource's own synchronization.
+    pub(crate) fn observe(&self) -> u64 {
+        self.state.lock().generation
+    }
+
+    pub(crate) fn has_waiters(&self) -> bool {
+        self.state.lock().waiters.is_some()
+    }
+
+    /// A fence the next notification signals. None means capacity changed after the sample:
+    /// retry instead of waiting. Called with no resource lock held. The recheck after allocation
+    /// closes both the failed-check/registration and the fence-allocation/release races. A
+    /// waiter that goes away leaves its entry for the next notification to signal.
+    pub(crate) fn wait(&self, observed: u64) -> Result<Option<Fence>> {
+        if self.observe() != observed {
+            return Ok(None);
+        }
+        let fence = fence::independent(&self.fences)?;
+        let mut entry = KBox::new(
+            Entry { fence: fence.clone(), next: None },
+            GFP_KERNEL,
+        )?;
+        let mut state = self.state.lock();
+        if state.generation != observed {
+            return Ok(None);
+        }
+        entry.next = state.waiters.take();
+        state.waiters = Some(entry);
+        Ok(Some(Fence::from_fence(&fence)))
+    }
+
+    /// Blocks until a notification after `observed`, for at most `timeout_ms`. For callers
+    /// outside the scheduler that own no other wait.
+    pub(crate) fn block(&self, observed: u64, timeout_ms: u32, interruptible: bool) -> Result {
+        let Some(fence) = self.wait(observed)? else { return Ok(()); };
+        // SAFETY: The owned fence keeps the raw DMA fence alive throughout the wait.
+        let waited = unsafe {
+            kernel::bindings::dma_fence_wait_timeout(
+                fence.raw(),
+                interruptible,
+                kernel::time::msecs_to_jiffies(timeout_ms) as isize,
+            )
+        };
+        if waited < 0 {
+            return Err(Error::from_errno(waited as i32));
+        }
+        Ok(())
+    }
+
+    /// Wakes every waiter of this domain. Call after dropping the resource's own lock; fence
+    /// callbacks run outside the domain lock.
+    pub(crate) fn notify(&self) {
+        let mut waiters = {
+            let mut state = self.state.lock();
+            state.generation = state.generation.wrapping_add(1);
+            state.waiters.take()
+        };
+        while let Some(mut entry) = waiters {
+            waiters = entry.next.take();
+            entry.fence.signal();
+        }
+    }
+}

drivers/gpu/drm/asahi/g17/capacity.rs

@@ -55,6 +55,11 @@ impl kernel::debugfs::Writer for View {
             let state = self.0.state.lock();
             let Some(firmware) = state.as_deref() else { return writeln!(f, "online=0"); };
             writeln!(f, "online=1")?;
+            let (converted, published) = super::sync_input::counters();
+            writeln!(f, "fw_inputs converted={} published={} native={} pending={}",
+                converted, published, super::sync_input::native_count(),
+                super::sync_input::pending_count())?;
+            writeln!(f, "compute_admission_demand={}", self.0.compute_demand.load(Ordering::Acquire))?;
             firmware.queues.write_capacity(f)?;
             firmware.pool.write_capacity(f)?;
             writeln!(f, "render_pool_ids used={} total=127", firmware.render_ids.used())?;

drivers/gpu/drm/asahi/g17/channel.rs

@@ -138,6 +138,7 @@ impl Rings {
             record.opcode().asserts_gpu_power(),
             next,
         )?;
+        super::trace::message(super::trace::CONTROL, record.opcode() as u64);
         Ok(next)
     }
 }